From 2e6707477beb94ccc18a9e49a44236fc4557b6ae Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 001/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From 8aa7baa5b8da9cd5b178ce01583b9c58bae63b9a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 002/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From fd5de2285141cfe57a6425a98a7890582c7aef1f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 003/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From 3cf37bc109bcb5f24eb3bc5b0d973a7c0894fcf3 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 004/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From b0e98c0eb22fb92aff271ec6003a0ef8d7ee0ffe Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 005/140] Added PubsubConsumer class --- .../clients/consumer/PubsubConsumer.java | 663 ++++++++++++++++++ 1 file changed, 663 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..12bb1ec5 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,663 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.cients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } +} \ No newline at end of file From 9642d5db085408ab1316c3ae33ffdf989e69442f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 006/140] New maven setup for mapped api --- pubsub-mapped-api/pom.xml | 49 +++++++ .../clients/consumer/PubsubConsumer.java | 134 +++++++++++++++++- .../clients/producer/PubsubProducer.java | 2 +- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 0 .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 10 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 pubsub-mapped-api/pom.xml rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/consumer/PubsubConsumer.java (83%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/PubsubProducer.java (99%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/PubsubProducerTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml new file mode 100644 index 00000000..1bb14363 --- /dev/null +++ b/pubsub-mapped-api/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + com.google.pubsub + pubsub-mapped-api + jar + 1.0-SNAPSHOT + MappedApi + http://maven.apache.org + + + junit + junit + 4.12 + + + org.apache.kafka + kafka_2.10 + 0.10.0.0 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.slf4j + slf4j-api + 1.7.21 + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java similarity index 83% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 7d6e0477..39e1ac87 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -11,7 +11,60 @@ * specific language governing permissions and limitations under the License. */ -package com.google.kafka.cients.consumer; +package com.google.kafka.clients.consumer; + +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.network.ChannelBuilder; +import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -106,7 +159,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { + /* try { log.debug("Starting the Kafka consumer"); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); @@ -145,9 +198,82 @@ private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, D this.keyDeserializer = keyDeserializer; } if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; } - } // left off at kafka's line 645 + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ } /** diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java similarity index 99% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 2077a3c1..5f7d3507 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -85,7 +85,7 @@ public class PubsubProducer implements Producer { private final ProducerConfig producerConfig; private final long maxBlockTimeMs; private final int requestTimeoutMs; - private final ProducerInterceptors interceptors + private final ProducerInterceptors interceptors; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 7104856eac1ffe36c2b64b4baf2c9c443a24b8af Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 007/140] scratching some of the mapped api to make it more pub/sub --- pubsub-mapped-api/pom.xml | 20 + .../clients/consumer/PubsubConsumer.java | 119 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 6 files changed, 24 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1bb14363..ef3ab09f 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -33,6 +33,26 @@ slf4j-log4j12 1.7.21 + + io.grpc + grpc-all + 1.0.1 + + + io.grpc + grpc-netty + 1.0.1 + + + io.grpc + grpc-protobuf + 1.0.1 + + + io.grpc + grpc-stub + 1.0.1 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 39e1ac87..1149cc4c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -159,121 +159,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -326,7 +212,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From 9b625833ee4b94d2ee3797abcddef04485364c41 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 008/140] Implementing publisher/producer using grpc backend. --- pubsub-mapped-api/pom.xml | 64 +- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../producer/PubsubProducerConfig.java | 85 ++ .../com/google/pubsub/common/PubsubUtils.java | 46 + .../src/main/resources/log4j.properties | 7 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 11 files changed, 1061 insertions(+), 1852 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java create mode 100644 pubsub-mapped-api/src/main/resources/log4j.properties delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index ef3ab09f..916913ca 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,6 +8,11 @@ MappedApi http://maven.apache.org + + com.google.pubsub + cloud-pubsub-client + 0.2-EXPERIMENTAL + junit junit @@ -16,7 +21,7 @@ org.apache.kafka kafka_2.10 - 0.10.0.0 + 0.10.1.1 org.apache.commons @@ -33,6 +38,11 @@ slf4j-log4j12 1.7.21 + + log4j + log4j + 1.2.17 + io.grpc grpc-all @@ -55,10 +65,62 @@ + + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + package + + shade + + + false + + + + com.google.pubsub.clients.ClientMain + + + mapped-api + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + + com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:0.14.0:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto/ + + + + + compile + compile-custom + + + + org.apache.maven.plugins maven-compiler-plugin + 3.6.1 1.8 1.8 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 1149cc4c..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,30 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -33,7 +34,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -68,608 +68,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java new file mode 100644 index 00000000..aa812494 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients.producer; + +import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.serialization.Serializer; + +public class PubsubProducerConfig extends AbstractConfig { + private static final ConfigDef CONFIG; + + public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + + public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + + public static final String BATCH_SIZE_CONFIG = "batch.size"; + private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; + + public static final String ACKS_CONFIG = "acks"; + private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project to use with the publisher."; + + static { + CONFIG = + new ConfigDef() + .define( + KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) + .define( + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + } + + PubsubProducerConfig(Map properties) { + super(CONFIG, properties); + } + + public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + newConfigs.putAll(configs); + if (keySerializer != null) { + newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newConfigs.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newConfigs; + } + + public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + Properties newProperties = new Properties(); + if (keySerializer != null) { + newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newProperties.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newProperties; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/resources/log4j.properties b/pubsub-mapped-api/src/main/resources/log4j.properties new file mode 100644 index 00000000..ff0cb5bd --- /dev/null +++ b/pubsub-mapped-api/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=DEBUG, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + +log4j.org.apache.kafka = OFF +log4j.logger.io.grpc = OFF +log4j.logger.io.netty = OFF \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From 342ef52fba1f9bb909493f5f81655e359f0ad7c0 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 009/140] Add details to simplified producer --- pubsub-mapped-api/pom.xml | 5 ++ .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../producer/PubsubProducerConfig.java | 6 +- .../internals/PubsubFutureRecordMetadata.java | 38 +++++++++++++ 4 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 916913ca..38b17362 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.cloud + google-cloud-pubsub + 0.9.2-alpha + junit junit diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index aa812494..5cee6425 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -43,6 +43,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; + private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + static { CONFIG = new ConfigDef() @@ -52,7 +55,8 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java new file mode 100644 index 00000000..88a5fd90 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +/*package com.google.pubsub.clients.producer.internals; + +private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; + } + + public boolean isCancelled() { + return false; + } + + public boolean isDone() { + return false; + } + + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; + } + + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +}*/ \ No newline at end of file From 2c61497991d50c5b6f9d31a329419deaa0cdc19a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 010/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 12 +++ .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 56 +++++++++++ .../pubsub/clients/ProducerThreadPool.java | 72 ++++++++++++++ .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 12 ++- ...ubsubUtils.java => PubsubChannelUtil.java} | 35 +++++-- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 8 files changed, 228 insertions(+), 178 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename pubsub-mapped-api/src/main/java/com/google/pubsub/common/{PubsubUtils.java => PubsubChannelUtil.java} (66%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 38b17362..99f6ff22 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,6 +23,18 @@ junit 4.12 + + org.powermock + powermock-module-junit4-legacy + 1.7.0RC2 + test + + + org.powermock + powermock-api-easymock + 1.7.0RC2 + test + org.apache.kafka kafka_2.10 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java new file mode 100644 index 00000000..092ac653 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -0,0 +1,56 @@ +package com.google.pubsub.clients; + +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; + + +public class ProducerThread implements Runnable { + private String command; + private PubsubProducer producer; + private String topic; + private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + + public ProducerThread(String s, Properties props, String topic) { + this.command = s; + this.producer = new PubsubProducer<>(props); + this.topic = topic; + } + + public void run() { + log.info("Start running the command"); + processCommand(); + log.info("End running the command"); + } + + private void processCommand() { + try { + ProducerRecord msg = new ProducerRecord(topic, "message" + command); + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } + } + ); + Thread.sleep(5000); + producer.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public String toString() { + return command; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java new file mode 100644 index 00000000..b37cba9e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -0,0 +1,72 @@ +package com.google.pubsub.clients; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class ProducerThreadPool { + private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); + + public static void main(String[] args) { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); + threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread t, Throwable e) { + log.error(t + " throws exception: " + e); + } + }); + + //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); + ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); + + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + + /* props.putAll(new ImmutableMap.Builder<>() + .put("max.block.ms", "30000") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("bootstrap.servers", "104.198.72.101:9092") + .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB + // 10M, high enough to allow for duration to control batching + .put("batch.size", Integer.toString(10 * 1000 * 1000)) + .put("linger.ms", 10) + .build() + );*/ + + for (int i = 0; i < 20; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0]); + executor.execute(worker); + } + + executor.shutdown(); + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + log.error("Executor did not terminate"); + } + } + } catch (InterruptedException ie) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..bcf86059 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null()); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 88a5fd90..771fcb82 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -13,9 +13,15 @@ * the License. */ -/*package com.google.pubsub.clients.producer.internals; +package com.google.pubsub.clients.producer.internals; -private static class PubsubFutureRecordMetadata implements Future { +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.producer.RecordMetadata; + +public class PubsubFutureRecordMetadata implements Future { public boolean cancel(boolean b) { return false; } @@ -35,4 +41,4 @@ public RecordMetadata get() throws InterruptedException, ExecutionException { public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } -}*/ \ No newline at end of file +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 10a5599e..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,31 +16,46 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; import io.grpc.auth.ClientAuthInterceptor; import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.concurrent.Executors; -public class PubsubUtils { +public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); + + private ManagedChannel channel; + private CallCredentials callCredentials; + + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + callCredentials = MoreCallCredentials.from(credentials); + channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + } + + public CallCredentials callCredentials() { + return callCredentials; + } + + public Channel channel() { + return channel; + } + + public void closeChannel() { + channel.shutdown(); } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From b0329b0d4aaf3a62b00ab1c1f379c317a551ec60 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 011/140] Working on unit tests, need to mock the publisher calls --- pubsub-mapped-api/pom.xml | 17 ++--- .../google/pubsub/clients/ProducerThread.java | 26 ++++---- .../pubsub/clients/ProducerThreadPool.java | 10 +-- .../clients/producer/PubsubProducer.java | 30 ++------- .../producer/PubsubProducerConfig.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- .../pubsub/common/PubsubChannelUtilTest.java | 51 +++++++++++++++ 8 files changed, 134 insertions(+), 77 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 99f6ff22..1fdd87b7 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,18 +23,6 @@ junit 4.12 - - org.powermock - powermock-module-junit4-legacy - 1.7.0RC2 - test - - - org.powermock - powermock-api-easymock - 1.7.0RC2 - test - org.apache.kafka kafka_2.10 @@ -80,6 +68,11 @@ grpc-stub 1.0.1 + + org.mockito + mockito-all + 2.0.2-beta + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 092ac653..e075ae36 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -30,19 +30,21 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord(topic, "message" + command); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } } - } - } - ); + ); + } Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index b37cba9e..4b168cc0 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -33,15 +33,15 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 1) + .put("batch.size", 20) .put("linger.ms", 1) .build() ); /* props.putAll(new ImmutableMap.Builder<>() .put("max.block.ms", "30000") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") .put("bootstrap.servers", "104.198.72.101:9092") .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB @@ -49,9 +49,9 @@ public void uncaughtException(Thread t, Throwable e) { .put("batch.size", Integer.toString(10 * 1000 * 1000)) .put("linger.ms", 10) .build() - );*/ + ); */ - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 1; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index bcf86059..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -273,7 +251,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null()); + doSend(request, null, null); } } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 5cee6425..ff3e6139 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -76,8 +76,8 @@ public static Map addSerializerToConfig(Map conf } public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap(); Properties newProperties = new Properties(); + newProperties.putAll(properties); if (keySerializer != null) { newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java new file mode 100644 index 00000000..2c5ad730 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.common; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PubsubChannelUtilTest { + + private static PubsubChannelUtil channelUtil; + + @BeforeClass + public static void setUp() throws IOException { + channelUtil = new PubsubChannelUtil(); + } + + @AfterClass + public static void tearDown() { + channelUtil.closeChannel(); + } + + @Test + public void testGetCallCredentials() throws IOException { + assertNotNull(channelUtil.callCredentials()); + channelUtil.closeChannel(); + } + + @Test + public void testGetChannel() { + assertNotNull(channelUtil.channel()); + } +} From fe0911d204f297853faf6cf4e58dc636f33de327 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 012/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 6 +- .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../producer/PubsubProducerConfig.java | 10 +- .../pubsub/common/PubsubChannelUtil.java | 4 - .../producer/PubsubProducerConfigTest.java | 59 +++++++++++ .../clients/producer/PubsubProducerTest.java | 35 +----- 6 files changed, 166 insertions(+), 48 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index e075ae36..2a3bb585 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -2,6 +2,7 @@ import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; +import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; @@ -18,7 +19,10 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic) { this.command = s; - this.producer = new PubsubProducer<>(props); + //this.producer = new PubsubProducer<>(props); + this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) + .batchSize(props.getProperty("batch.size")) + .) this.topic = topic; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index ff3e6139..f03c017f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -46,6 +46,10 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final boolean DEFAULT_ACKS = true; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + static { CONFIG = new ConfigDef() @@ -53,10 +57,10 @@ public class PubsubProducerConfig extends AbstractConfig { KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java new file mode 100644 index 00000000..000536cd --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -0,0 +1,59 @@ +package com.google.pubsub.clients.producer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Properties; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.serialization.Serializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + + +public class PubsubProducerConfigTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testSuccessAllConfigsProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "unit-test-project") + .build() + ); + + PubsubProducerConfig testConfig = new PubsubProducerConfig(props); + + assertEquals("Project config equals unit-test-project.", "unit-test-project", testConfig.getString(PubsubProducerConfig.PROJECT_CONFIG)); + assertNotNull("Key serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + assertNotNull("Value serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + } + + @Test + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "unit-test-project"); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + + } + + @Test + public void testNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From 191ab7a75cc1c8fcd25b9a0a96f2ecb59b5e7593 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 013/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 16 +++++++++------- .../pubsub/clients/ProducerThreadPool.java | 7 ++++--- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 2a3bb585..d1ae43fa 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,6 +1,8 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; @@ -17,12 +19,12 @@ public class ProducerThread implements Runnable { private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); - public ProducerThread(String s, Properties props, String topic) { + public ProducerThread(String s, Properties props, String topic) throws IOException { this.command = s; - //this.producer = new PubsubProducer<>(props); - this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) - .batchSize(props.getProperty("batch.size")) - .) + this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + .batchSize(Integer.parseInt(props.getProperty("batch.size"))) + .isAcks(props.getProperty("acks").matches("1|all")) + .build(); this.topic = topic; } @@ -34,8 +36,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 4b168cc0..edb9707b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -1,5 +1,6 @@ package com.google.pubsub.clients; +import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -15,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -33,8 +34,8 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 20) - .put("linger.ms", 1) + .put("batch.size", "1") + .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From acd7a2a98c182de71674f41960a37d97e45f838d Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 08:32:09 -0800 Subject: [PATCH 014/140] Mapped publisher task for load testing. --- .../clients/mapped/MappedPublisherTask.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java new file mode 100644 index 00000000..868fe2d3 --- /dev/null +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -0,0 +1,71 @@ +package com.google.pubsub.clients.mapped; + +import com.beust.jcommander.JCommander; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.pubsub.clients.common.LoadTestRunner; +import com.google.pubsub.clients.common.MetricsHandler; +import com.google.pubsub.clients.common.MetricsHandler.MetricName; +import com.google.pubsub.clients.common.Task; +import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.flic.common.LoadtestProto.StartRequest; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer + * interface + */ +public class MappedPublisherTask extends Task { + + private static final Logger log = LoggerFactory.getLogger(MappedPublisherTask.class); + private final String topic; + private final String payload; + private final int batchSize; + private final PubsubProducer publisher; + + @SuppressWarnings("unchecked") + private MappedPublisherTask(StartRequest request) throws IOException { + super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); + this.topic = request.getTopic(); + this.payload = LoadTestRunner.createMessage(request.getMessageSize()); + this.batchSize = request.getPublishBatchSize(); + + this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), + new StringSerializer()) + .batchSize(this.batchSize) + .isAcks(true) + .build(); + } + + public static void main(String[] args) throws Exception { + LoadTestRunner.Options options = new LoadTestRunner.Options(); + new JCommander(options, args); + LoadTestRunner.run(options, MappedPublisherTask::new); + } + + @Override + public ListenableFuture doRun() { + SettableFuture result = SettableFuture.create(); + AtomicInteger messagesToSend = new AtomicInteger(batchSize); + AtomicInteger messagesSentSuccess = new AtomicInteger(batchSize); + for (int i = 0; i < batchSize; i++) { + publisher.send( + new ProducerRecord<>(topic, null, System.currentTimeMillis(), null, payload), + (metadata, exception) -> { + if (exception != null) { + messagesSentSuccess.decrementAndGet(); + log.error(exception.getMessage(), exception); + } + if (messagesToSend.decrementAndGet() == 0) { + result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + } + }); + } + return result; + } +} From e836cf007976969a7aabe0ae69b84816eb09fd4a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 13:32:36 -0800 Subject: [PATCH 015/140] Fixing dependencies between load-test-framework and mapped-api --- load-test-framework/pom.xml | 5 + .../clients/mapped/MappedPublisherTask.java | 3 +- .../pubsub/flic/controllers/Client.java | 5 +- pubsub-mapped-api/pom.xml | 17 ++- .../clients/producer/PubsubProducer.java | 3 +- .../pubsub/common/PubsubChannelUtil.java | 17 ++- .../clients/producer/PubsubProducerTest.java | 114 ++++++++++++++---- 7 files changed, 128 insertions(+), 36 deletions(-) diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 582821bd..1a443a4e 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.pubsub + pubsub-mapped-api + 1.0-SNAPSHOT + junit junit diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java index 868fe2d3..a9410668 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -4,7 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler; import com.google.pubsub.clients.common.MetricsHandler.MetricName; import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.producer.PubsubProducer; @@ -29,7 +28,7 @@ public class MappedPublisherTask extends Task { private final PubsubProducer publisher; @SuppressWarnings("unchecked") - private MappedPublisherTask(StartRequest request) throws IOException { + private MappedPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); this.payload = LoadTestRunner.createMessage(request.getMessageSize()); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index a246675d..78285efe 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -116,6 +116,8 @@ public static String getTopicSuffix(ClientType clientType) { case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; + case MAPPED_PUBLISHER: + return "mapped"; } return null; } @@ -301,7 +303,8 @@ public enum ClientType { CPS_GCLOUD_PYTHON_PUBLISHER, CPS_VTK_JAVA_PUBLISHER, KAFKA_PUBLISHER, - KAFKA_SUBSCRIBER; + KAFKA_SUBSCRIBER, + MAPPED_PUBLISHER; public boolean isCpsPublisher() { switch (this) { diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1fdd87b7..d9e7639a 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -28,6 +28,21 @@ kafka_2.10 0.10.1.1 + + org.powermock + powermock-module-junit4 + 1.6.6 + + + org.powermock + powermock-api-mockito + 1.6.6 + + + org.powermock + powermock-api-easymock + 1.6.6 + org.apache.commons commons-lang3 @@ -71,7 +86,7 @@ org.mockito mockito-all - 2.0.2-beta + 1.9.5 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..48eba14c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -27,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -360,7 +359,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..a6d14269 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,8 +16,6 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; @@ -27,10 +25,12 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { - + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +41,15 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault() + .createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 32555be2..de8c1ae6 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,45 +16,109 @@ */ package com.google.pubsub.clients.producer; -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.powermock.api.easymock.PowerMock.createMock; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import java.io.IOException; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - +//@RunWith(PowerMockRunner.class) +//@PrepareForTest(PublisherFutureStub.class) public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; + private static final String PROJECT = "unit-test-proj"; + private static final StringSerializer testSerializer = new StringSerializer(); + private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); + + private static PublisherFutureStub mockStub; + @Mock private static PubsubChannelUtil mockChannelUtil; + + private boolean channelIsClosed; + + + @Before + public void setUp() { + // mockStub = createMock(PublisherFutureStub.class); + MockitoAnnotations.initMocks(this); + channelIsClosed = false; + + Mockito.doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + return channelIsClosed = true; + } + }).when(mockChannelUtil).closeChannel(); + } /* send() tests */ @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub + public void testSendPublisherClosed() throws IOException { + /* PubsubProducer testProducer = getNewProducer(); + testProducer.close(); + + try { + testProducer.send(testRecord); + fail("Should have thrown a runtime exception."); + } catch (RuntimeException expected) { + assertTrue("Channel should be closed.", channelIsClosed); + }*/ + } + + @Test + public void testSendRecordTooLarge() { + + } + + @Test + public void testSendBatchFull() { + + } + + /* This happens when the batch size is > 1 and not enough messages are batched */ + @Test + public void testSendMessageNotSentYet() { + + } + + /* flush() tests */ + @Test + public void testFlushMessagesSent() { + + } + + /* close() tests */ + @Test + public void testCloseTimeoutLessThanZero() { + + } + + @Test + public void testCloseChannelCloseSuccessful() { - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal } - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); + private PubsubProducer getNewProducer() throws IOException { - return new PubsubProducer(props); + return new Builder<>(PROJECT, testSerializer, testSerializer) + .publisherFutureStub(mockStub) + .pubsubChannelUtil(mockChannelUtil) + .build(); } } From 0000acf7fb67fd83406f572a7e86e33abaadc17b Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 16:49:51 -0800 Subject: [PATCH 016/140] Fixed unit tests to be compatible with my load test changes. --- .../pubsub/clients/mapped/MappedPublisherTask.java | 1 - .../main/java/com/google/pubsub/flic/Driver.java | 13 ++++++++++++- .../com/google/pubsub/flic/controllers/Client.java | 3 +++ .../google/pubsub/flic/output/SheetsService.java | 2 ++ .../pubsub/flic/output/SheetsServiceTest.java | 5 ++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java index a9410668..ffb01012 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -8,7 +8,6 @@ import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.producer.PubsubProducer; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.StringSerializer; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 04dc5af6..3fa0803f 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -130,6 +130,13 @@ public class Driver { ) private int kafkaSubscriberCount = 0; + @Parameter( + names = {"--mapped_publisher_count"}, + description = "Number of mapped publishers to start." + ) + + private int mappedPublisherCount = 0; + @Parameter( names = {"--message_size", "-m"}, description = "Message size in bytes (only when publishing messages).", @@ -372,6 +379,10 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } + if (mappedPublisherCount > 0) { + clientParamsMap.put( + new ClientParams(ClientType.MAPPED_PUBLISHER, null), mappedPublisherCount); + } if (kafkaPublisherCount > 0) { clientParamsMap.put( new ClientParams(ClientType.KAFKA_PUBLISHER, null), kafkaPublisherCount); @@ -402,7 +413,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + mappedPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index 78285efe..68f34b47 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -198,6 +198,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case MAPPED_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -334,6 +335,7 @@ public boolean isPublisher() { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: case KAFKA_PUBLISHER: + case MAPPED_PUBLISHER: return true; default: return false; @@ -347,6 +349,7 @@ public ClientType getSubscriberType() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case MAPPED_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 1e47aaec..08718d2c 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -170,6 +170,8 @@ public List>> getValuesList(Map> types = new HashMap<>(); int expectedCpsCount = 0; int expectedKafkaCount = 0; + int expectedMappedCount = 0; Map paramsMap = new HashMap<>(); for (ClientType type : ClientType.values()) { paramsMap.put(new ClientParams(type, ""), 1); @@ -44,8 +45,10 @@ public void testClientSwitch() { expectedCpsCount++; } else if (type.toString().startsWith("kafka")) { expectedKafkaCount++; + } else if (type.toString().startsWith("mapped")) { + expectedMappedCount++; } else { - fail("ClientType toString didn't start with cps or kafka"); + fail("ClientType toString didn't start with cps, mapped, or kafka"); } } types.put("zone-test", paramsMap); From 7a28f86332d041cd58c0e1427c4104aa8c3fa99d Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:19:31 -0800 Subject: [PATCH 017/140] Updated pom for load tests --- load-test-framework/flic.iml | 117 ----------------------------------- load-test-framework/pom.xml | 5 ++ 2 files changed, 5 insertions(+), 117 deletions(-) delete mode 100644 load-test-framework/flic.iml diff --git a/load-test-framework/flic.iml b/load-test-framework/flic.iml deleted file mode 100644 index 6826b98d..00000000 --- a/load-test-framework/flic.iml +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 1a443a4e..21047f54 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -155,6 +155,11 @@ zkclient 0.7 + + org.apache.kafka + kafka-clients + 0.10.0.0 + From 590f2dc1aa4cd768122b448f89507697933da2a4 Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 018/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From d67ea1761fcfbf22403917f31509f1e629bc1482 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 019/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From 6aa8f732c50ac52b75afc6d4f44c4425b9d7c241 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 020/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From b299edf1005773e2624b7f702d88b47bedd0c67a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 021/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From f94caad76c8cf59f8888b878b2632674c3b07001 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 022/140] Added PubsubConsumer class --- load-test-framework/flic.iml | 117 --- .../clients/consumer/PubsubConsumer.java | 743 +++++++++++++++--- 2 files changed, 647 insertions(+), 213 deletions(-) delete mode 100644 load-test-framework/flic.iml diff --git a/load-test-framework/flic.iml b/load-test-framework/flic.iml deleted file mode 100644 index 6826b98d..00000000 --- a/load-test-framework/flic.iml +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 42b3f7db..12bb1ec5 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -10,103 +10,654 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; +package com.google.kafka.cients.consumer; public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; // this might need to be an /internal class - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - // currentThread holds the threadId of the current thread accessing PubsubConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file From 204a057548f33f3b1646c935064d31677baceefc Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 023/140] New maven setup for mapped api --- pubsub-mapped-api/pom.xml | 49 +++++++ .../clients/consumer/PubsubConsumer.java | 136 +++++++++++++++++- .../clients/producer/PubsubProducer.java | 2 +- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 0 .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 10 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 pubsub-mapped-api/pom.xml rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/consumer/PubsubConsumer.java (83%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/PubsubProducer.java (99%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/PubsubProducerTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml new file mode 100644 index 00000000..1bb14363 --- /dev/null +++ b/pubsub-mapped-api/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + com.google.pubsub + pubsub-mapped-api + jar + 1.0-SNAPSHOT + MappedApi + http://maven.apache.org + + + junit + junit + 4.12 + + + org.apache.kafka + kafka_2.10 + 0.10.0.0 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.slf4j + slf4j-api + 1.7.21 + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java similarity index 83% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 12bb1ec5..cadfbf1b 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -10,7 +10,64 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ +<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; +======= + +package com.google.kafka.clients.consumer; + +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.network.ChannelBuilder; +import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -105,7 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { + /* try { log.debug("Starting the Kafka consumer"); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); @@ -144,9 +201,82 @@ private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, D this.keyDeserializer = keyDeserializer; } if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; } - } // left off at kafka's line 645 + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ } /** diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java similarity index 99% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 2077a3c1..5f7d3507 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -85,7 +85,7 @@ public class PubsubProducer implements Producer { private final ProducerConfig producerConfig; private final long maxBlockTimeMs; private final int requestTimeoutMs; - private final ProducerInterceptors interceptors + private final ProducerInterceptors interceptors; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 6da427b8fb3bc84e5e14a0ad9830d9630bd175bc Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 024/140] scratching some of the mapped api to make it more pub/sub --- pubsub-mapped-api/pom.xml | 20 + .../clients/consumer/PubsubConsumer.java | 119 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 6 files changed, 24 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1bb14363..ef3ab09f 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -33,6 +33,26 @@ slf4j-log4j12 1.7.21 + + io.grpc + grpc-all + 1.0.1 + + + io.grpc + grpc-netty + 1.0.1 + + + io.grpc + grpc-protobuf + 1.0.1 + + + io.grpc + grpc-stub + 1.0.1 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index cadfbf1b..72eb4413 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -14,7 +14,7 @@ package com.google.kafka.cients.consumer; ======= -package com.google.kafka.clients.consumer; +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -162,121 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -329,7 +215,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From 29863e99eba82dfa0fa8d9c89be383c398ae28b7 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 025/140] Implementing publisher/producer using grpc backend. --- pubsub-mapped-api/pom.xml | 64 +- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../producer/PubsubProducerConfig.java | 85 ++ .../com/google/pubsub/common/PubsubUtils.java | 46 + .../src/main/resources/log4j.properties | 7 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 11 files changed, 1061 insertions(+), 1852 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java create mode 100644 pubsub-mapped-api/src/main/resources/log4j.properties delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index ef3ab09f..916913ca 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,6 +8,11 @@ MappedApi http://maven.apache.org + + com.google.pubsub + cloud-pubsub-client + 0.2-EXPERIMENTAL + junit junit @@ -16,7 +21,7 @@ org.apache.kafka kafka_2.10 - 0.10.0.0 + 0.10.1.1 org.apache.commons @@ -33,6 +38,11 @@ slf4j-log4j12 1.7.21 + + log4j + log4j + 1.2.17 + io.grpc grpc-all @@ -55,10 +65,62 @@ + + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + package + + shade + + + false + + + + com.google.pubsub.clients.ClientMain + + + mapped-api + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + + com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:0.14.0:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto/ + + + + + compile + compile-custom + + + + org.apache.maven.plugins maven-compiler-plugin + 3.6.1 1.8 1.8 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 72eb4413..e5abef92 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,14 +1,16 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ <<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; @@ -16,18 +18,17 @@ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -36,7 +37,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -71,608 +71,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java new file mode 100644 index 00000000..aa812494 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients.producer; + +import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.serialization.Serializer; + +public class PubsubProducerConfig extends AbstractConfig { + private static final ConfigDef CONFIG; + + public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + + public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + + public static final String BATCH_SIZE_CONFIG = "batch.size"; + private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; + + public static final String ACKS_CONFIG = "acks"; + private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project to use with the publisher."; + + static { + CONFIG = + new ConfigDef() + .define( + KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) + .define( + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + } + + PubsubProducerConfig(Map properties) { + super(CONFIG, properties); + } + + public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + newConfigs.putAll(configs); + if (keySerializer != null) { + newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newConfigs.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newConfigs; + } + + public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + Properties newProperties = new Properties(); + if (keySerializer != null) { + newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newProperties.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newProperties; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/resources/log4j.properties b/pubsub-mapped-api/src/main/resources/log4j.properties new file mode 100644 index 00000000..ff0cb5bd --- /dev/null +++ b/pubsub-mapped-api/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=DEBUG, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + +log4j.org.apache.kafka = OFF +log4j.logger.io.grpc = OFF +log4j.logger.io.netty = OFF \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From 8838e041535f099dace30cbb821fe1170f4655f1 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 026/140] Add details to simplified producer --- pubsub-mapped-api/pom.xml | 5 ++ .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../producer/PubsubProducerConfig.java | 6 +- .../internals/PubsubFutureRecordMetadata.java | 38 +++++++++++++ 4 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 916913ca..38b17362 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.cloud + google-cloud-pubsub + 0.9.2-alpha + junit junit diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index aa812494..5cee6425 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -43,6 +43,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; + private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + static { CONFIG = new ConfigDef() @@ -52,7 +55,8 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java new file mode 100644 index 00000000..88a5fd90 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +/*package com.google.pubsub.clients.producer.internals; + +private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; + } + + public boolean isCancelled() { + return false; + } + + public boolean isDone() { + return false; + } + + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; + } + + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +}*/ \ No newline at end of file From 30ffa3b8ee1aa08cc5a11c046b9310bc3b5a6b56 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 027/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 12 +++ .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 56 +++++++++++ .../pubsub/clients/ProducerThreadPool.java | 72 ++++++++++++++ .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 12 ++- ...ubsubUtils.java => PubsubChannelUtil.java} | 35 +++++-- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 8 files changed, 228 insertions(+), 178 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename pubsub-mapped-api/src/main/java/com/google/pubsub/common/{PubsubUtils.java => PubsubChannelUtil.java} (66%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 38b17362..99f6ff22 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,6 +23,18 @@ junit 4.12 + + org.powermock + powermock-module-junit4-legacy + 1.7.0RC2 + test + + + org.powermock + powermock-api-easymock + 1.7.0RC2 + test + org.apache.kafka kafka_2.10 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java new file mode 100644 index 00000000..092ac653 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -0,0 +1,56 @@ +package com.google.pubsub.clients; + +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; + + +public class ProducerThread implements Runnable { + private String command; + private PubsubProducer producer; + private String topic; + private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + + public ProducerThread(String s, Properties props, String topic) { + this.command = s; + this.producer = new PubsubProducer<>(props); + this.topic = topic; + } + + public void run() { + log.info("Start running the command"); + processCommand(); + log.info("End running the command"); + } + + private void processCommand() { + try { + ProducerRecord msg = new ProducerRecord(topic, "message" + command); + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } + } + ); + Thread.sleep(5000); + producer.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public String toString() { + return command; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java new file mode 100644 index 00000000..b37cba9e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -0,0 +1,72 @@ +package com.google.pubsub.clients; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class ProducerThreadPool { + private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); + + public static void main(String[] args) { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); + threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread t, Throwable e) { + log.error(t + " throws exception: " + e); + } + }); + + //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); + ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); + + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + + /* props.putAll(new ImmutableMap.Builder<>() + .put("max.block.ms", "30000") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("bootstrap.servers", "104.198.72.101:9092") + .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB + // 10M, high enough to allow for duration to control batching + .put("batch.size", Integer.toString(10 * 1000 * 1000)) + .put("linger.ms", 10) + .build() + );*/ + + for (int i = 0; i < 20; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0]); + executor.execute(worker); + } + + executor.shutdown(); + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + log.error("Executor did not terminate"); + } + } + } catch (InterruptedException ie) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..bcf86059 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null()); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 88a5fd90..771fcb82 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -13,9 +13,15 @@ * the License. */ -/*package com.google.pubsub.clients.producer.internals; +package com.google.pubsub.clients.producer.internals; -private static class PubsubFutureRecordMetadata implements Future { +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.producer.RecordMetadata; + +public class PubsubFutureRecordMetadata implements Future { public boolean cancel(boolean b) { return false; } @@ -35,4 +41,4 @@ public RecordMetadata get() throws InterruptedException, ExecutionException { public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } -}*/ \ No newline at end of file +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 10a5599e..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,31 +16,46 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; import io.grpc.auth.ClientAuthInterceptor; import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.concurrent.Executors; -public class PubsubUtils { +public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); + + private ManagedChannel channel; + private CallCredentials callCredentials; + + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + callCredentials = MoreCallCredentials.from(credentials); + channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + } + + public CallCredentials callCredentials() { + return callCredentials; + } + + public Channel channel() { + return channel; + } + + public void closeChannel() { + channel.shutdown(); } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From 1616cf3169d9b8b44192edc9e38f8b91fd713578 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 028/140] Working on unit tests, need to mock the publisher calls --- pubsub-mapped-api/pom.xml | 17 ++--- .../google/pubsub/clients/ProducerThread.java | 26 ++++---- .../pubsub/clients/ProducerThreadPool.java | 10 +-- .../clients/producer/PubsubProducer.java | 30 ++------- .../producer/PubsubProducerConfig.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- .../pubsub/common/PubsubChannelUtilTest.java | 51 +++++++++++++++ 8 files changed, 134 insertions(+), 77 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 99f6ff22..1fdd87b7 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,18 +23,6 @@ junit 4.12 - - org.powermock - powermock-module-junit4-legacy - 1.7.0RC2 - test - - - org.powermock - powermock-api-easymock - 1.7.0RC2 - test - org.apache.kafka kafka_2.10 @@ -80,6 +68,11 @@ grpc-stub 1.0.1 + + org.mockito + mockito-all + 2.0.2-beta + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 092ac653..e075ae36 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -30,19 +30,21 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord(topic, "message" + command); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } } - } - } - ); + ); + } Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index b37cba9e..4b168cc0 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -33,15 +33,15 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 1) + .put("batch.size", 20) .put("linger.ms", 1) .build() ); /* props.putAll(new ImmutableMap.Builder<>() .put("max.block.ms", "30000") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") .put("bootstrap.servers", "104.198.72.101:9092") .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB @@ -49,9 +49,9 @@ public void uncaughtException(Thread t, Throwable e) { .put("batch.size", Integer.toString(10 * 1000 * 1000)) .put("linger.ms", 10) .build() - );*/ + ); */ - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 1; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index bcf86059..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -273,7 +251,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null()); + doSend(request, null, null); } } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 5cee6425..ff3e6139 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -76,8 +76,8 @@ public static Map addSerializerToConfig(Map conf } public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap(); Properties newProperties = new Properties(); + newProperties.putAll(properties); if (keySerializer != null) { newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java new file mode 100644 index 00000000..2c5ad730 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.common; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PubsubChannelUtilTest { + + private static PubsubChannelUtil channelUtil; + + @BeforeClass + public static void setUp() throws IOException { + channelUtil = new PubsubChannelUtil(); + } + + @AfterClass + public static void tearDown() { + channelUtil.closeChannel(); + } + + @Test + public void testGetCallCredentials() throws IOException { + assertNotNull(channelUtil.callCredentials()); + channelUtil.closeChannel(); + } + + @Test + public void testGetChannel() { + assertNotNull(channelUtil.channel()); + } +} From f2d3ff974fa426d4b0447d924c14af074fb0a863 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 029/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 6 +- .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../producer/PubsubProducerConfig.java | 10 +- .../pubsub/common/PubsubChannelUtil.java | 4 - .../producer/PubsubProducerConfigTest.java | 59 +++++++++++ .../clients/producer/PubsubProducerTest.java | 35 +----- 6 files changed, 166 insertions(+), 48 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index e075ae36..2a3bb585 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -2,6 +2,7 @@ import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; +import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; @@ -18,7 +19,10 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic) { this.command = s; - this.producer = new PubsubProducer<>(props); + //this.producer = new PubsubProducer<>(props); + this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) + .batchSize(props.getProperty("batch.size")) + .) this.topic = topic; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index ff3e6139..f03c017f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -46,6 +46,10 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final boolean DEFAULT_ACKS = true; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + static { CONFIG = new ConfigDef() @@ -53,10 +57,10 @@ public class PubsubProducerConfig extends AbstractConfig { KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java new file mode 100644 index 00000000..000536cd --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -0,0 +1,59 @@ +package com.google.pubsub.clients.producer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Properties; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.serialization.Serializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + + +public class PubsubProducerConfigTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testSuccessAllConfigsProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "unit-test-project") + .build() + ); + + PubsubProducerConfig testConfig = new PubsubProducerConfig(props); + + assertEquals("Project config equals unit-test-project.", "unit-test-project", testConfig.getString(PubsubProducerConfig.PROJECT_CONFIG)); + assertNotNull("Key serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + assertNotNull("Value serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + } + + @Test + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "unit-test-project"); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + + } + + @Test + public void testNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From 4df5c833deaf00cd963c899eb04164aa16514da3 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 030/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 16 +++++++++------- .../pubsub/clients/ProducerThreadPool.java | 7 ++++--- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 2a3bb585..d1ae43fa 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,6 +1,8 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; @@ -17,12 +19,12 @@ public class ProducerThread implements Runnable { private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); - public ProducerThread(String s, Properties props, String topic) { + public ProducerThread(String s, Properties props, String topic) throws IOException { this.command = s; - //this.producer = new PubsubProducer<>(props); - this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) - .batchSize(props.getProperty("batch.size")) - .) + this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + .batchSize(Integer.parseInt(props.getProperty("batch.size"))) + .isAcks(props.getProperty("acks").matches("1|all")) + .build(); this.topic = topic; } @@ -34,8 +36,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 4b168cc0..edb9707b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -1,5 +1,6 @@ package com.google.pubsub.clients; +import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -15,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -33,8 +34,8 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 20) - .put("linger.ms", 1) + .put("batch.size", "1") + .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From 2c94968a5a6e51c58dd7ea69943649f7328dee39 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:46:08 -0800 Subject: [PATCH 031/140] Minor fixes to producer's classes --- .../pubsub/clients/consumer/PubsubConsumer.java | 3 --- .../pubsub/clients/producer/PubsubProducer.java | 2 +- .../com/google/pubsub/common/PubsubChannelUtil.java | 13 +++++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index e5abef92..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -12,9 +12,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ -<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java -package com.google.kafka.cients.consumer; -======= package com.google.pubsub.clients.consumer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..e9ca5753 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -360,7 +360,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..9088d5bf 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +44,14 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } From d6183b8aa17456ab8cfe4809c843e3c16ed59510 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 10:06:42 -0800 Subject: [PATCH 032/140] Omitting unit tests and unused classes --- .../internals/PubsubFutureRecordMetadata.java | 44 -------------- .../clients/producer/PubsubProducerTest.java | 60 ------------------- .../pubsub/common/PubsubChannelUtilTest.java | 51 ---------------- 3 files changed, 155 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java deleted file mode 100644 index 771fcb82..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.pubsub.clients.producer.internals; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.kafka.clients.producer.RecordMetadata; - -public class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index 32555be2..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - - /* send() tests */ - @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub - - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal - } - - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); - - return new PubsubProducer(props); - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java deleted file mode 100644 index 2c5ad730..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.pubsub.common; - -import static org.junit.Assert.assertNotNull; - -import java.io.IOException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PubsubChannelUtilTest { - - private static PubsubChannelUtil channelUtil; - - @BeforeClass - public static void setUp() throws IOException { - channelUtil = new PubsubChannelUtil(); - } - - @AfterClass - public static void tearDown() { - channelUtil.closeChannel(); - } - - @Test - public void testGetCallCredentials() throws IOException { - assertNotNull(channelUtil.callCredentials()); - channelUtil.closeChannel(); - } - - @Test - public void testGetChannel() { - assertNotNull(channelUtil.channel()); - } -} From 13546e169683eaffaaf24d1f29d08bc565823d3f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 16:19:17 -0800 Subject: [PATCH 033/140] Adding the mapped publisher to the loadtest framework. --- .../gce/mapped-publisher_startup_script.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh diff --git a/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh b/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh new file mode 100644 index 00000000..0b0631b5 --- /dev/null +++ b/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +####################################### +# Query GCE for a provided metadata field. +# See https://developers.google.com/compute/docs/metadata +# Globals: +# None +# Arguments: +# $1: The path to the metadata field to retrieve +# Returns: +# The value stored at the metadata field +####################################### +function metadata() { + curl --silent --show-error --header 'Metadata-Flavor: Google' \ + "http://metadata/computeMetadata/v1/${1}"; +} + +readonly TMP="$(mktemp -d)" +readonly BUCKET=$(metadata instance/attributes/bucket) + +[[ "${TMP}" != "" ]] || error mktemp failed + +# Download the loadtest binary to this machine and install Java 8. +/usr/bin/apt-get update +/usr/bin/apt-get install -y openjdk-8-jre-headless & PIDAPT=$! +/usr/bin/gsutil cp "gs://${BUCKET}/driver.jar" "${TMP}" + +wait $PIDAPT + +# Run the loadtest binary +java -Xmx5000M -cp ${TMP}/driver.jar com.google.pubsub.clients.mapped.MappedPublisherTask From 534725e89fb025de4a727b409bd4b7bb7d4e568a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 7 Mar 2017 11:06:36 -0800 Subject: [PATCH 034/140] Attempting to fix 'cannot load' error for publisher task. --- ...appedPublisherTask.java => CPSPublisherTask.java} | 11 +++++++---- .../src/main/java/com/google/pubsub/flic/Driver.java | 12 ++++++------ .../com/google/pubsub/flic/controllers/Client.java | 11 ++++++----- .../com/google/pubsub/flic/output/SheetsService.java | 3 ++- ...h => cps-mapped-java-publisher_startup_script.sh} | 4 ++-- .../google/pubsub/flic/output/SheetsServiceTest.java | 5 +---- 6 files changed, 24 insertions(+), 22 deletions(-) rename load-test-framework/src/main/java/com/google/pubsub/clients/mapped/{MappedPublisherTask.java => CPSPublisherTask.java} (89%) rename load-test-framework/src/main/resources/gce/{mapped-publisher_startup_script.sh => cps-mapped-java-publisher_startup_script.sh} (95%) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java similarity index 89% rename from load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java rename to load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index ffb01012..e100f069 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -18,16 +18,16 @@ * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer * interface */ -public class MappedPublisherTask extends Task { +public class CPSPublisherTask extends Task { - private static final Logger log = LoggerFactory.getLogger(MappedPublisherTask.class); + private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); private final String topic; private final String payload; private final int batchSize; private final PubsubProducer publisher; @SuppressWarnings("unchecked") - private MappedPublisherTask(StartRequest request) { + private CPSPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); this.payload = LoadTestRunner.createMessage(request.getMessageSize()); @@ -43,7 +43,7 @@ private MappedPublisherTask(StartRequest request) { public static void main(String[] args) throws Exception { LoadTestRunner.Options options = new LoadTestRunner.Options(); new JCommander(options, args); - LoadTestRunner.run(options, MappedPublisherTask::new); + LoadTestRunner.run(options, CPSPublisherTask::new); } @Override @@ -66,4 +66,7 @@ public ListenableFuture doRun() { } return result; } + + @Override + public void shutdown() { publisher.close(); } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 3fa0803f..6bcfc600 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -131,11 +131,11 @@ public class Driver { private int kafkaSubscriberCount = 0; @Parameter( - names = {"--mapped_publisher_count"}, - description = "Number of mapped publishers to start." + names = {"--cps_mapped_publisher_count"}, + description = "Number of cps mapped publishers to start." ) - private int mappedPublisherCount = 0; + private int cpsMappedPublisherCount = 0; @Parameter( names = {"--message_size", "-m"}, @@ -379,9 +379,9 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } - if (mappedPublisherCount > 0) { + if (cpsMappedPublisherCount > 0) { clientParamsMap.put( - new ClientParams(ClientType.MAPPED_PUBLISHER, null), mappedPublisherCount); + new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), cpsMappedPublisherCount); } if (kafkaPublisherCount > 0) { clientParamsMap.put( @@ -413,7 +413,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + mappedPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index 68f34b47..523a0cb6 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -116,7 +116,7 @@ public static String getTopicSuffix(ClientType clientType) { case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return "mapped"; } return null; @@ -198,7 +198,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -305,7 +305,7 @@ public enum ClientType { CPS_VTK_JAVA_PUBLISHER, KAFKA_PUBLISHER, KAFKA_SUBSCRIBER, - MAPPED_PUBLISHER; + CPS_MAPPED_JAVA_PUBLISHER; public boolean isCpsPublisher() { switch (this) { @@ -313,6 +313,7 @@ public boolean isCpsPublisher() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -335,7 +336,7 @@ public boolean isPublisher() { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: case KAFKA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -349,7 +350,7 @@ public ClientType getSubscriberType() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 08718d2c..a60d527a 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -78,6 +78,7 @@ private void fillClientCounts(Map> types) { cpsPublisherCount += (countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) != null) ? countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) : 0; cpsPublisherCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER): 0; cpsPublisherCount += (countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) : 0; + cpsPublisherCount += (countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) : 0; cpsSubscriberCount += (countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER): 0; cpsSubscriberCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER): 0; kafkaPublisherCount += (countMap.get(ClientType.KAFKA_PUBLISHER) != null) ? countMap.get(ClientType.KAFKA_PUBLISHER) : 0; @@ -170,7 +171,7 @@ public List>> getValuesList(Map> types = new HashMap<>(); int expectedCpsCount = 0; int expectedKafkaCount = 0; - int expectedMappedCount = 0; Map paramsMap = new HashMap<>(); for (ClientType type : ClientType.values()) { paramsMap.put(new ClientParams(type, ""), 1); @@ -45,10 +44,8 @@ public void testClientSwitch() { expectedCpsCount++; } else if (type.toString().startsWith("kafka")) { expectedKafkaCount++; - } else if (type.toString().startsWith("mapped")) { - expectedMappedCount++; } else { - fail("ClientType toString didn't start with cps, mapped, or kafka"); + fail("ClientType toString didn't start with cps or kafka"); } } types.put("zone-test", paramsMap); From fb53676616311e49fbfc7d2e7f26dbbdcddc7fb7 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 10:37:54 -0800 Subject: [PATCH 035/140] Fixed naming for cps_mapped_java_publisher --- .../src/main/java/com/google/pubsub/flic/Driver.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 6bcfc600..e1daa3c0 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -131,11 +131,11 @@ public class Driver { private int kafkaSubscriberCount = 0; @Parameter( - names = {"--cps_mapped_publisher_count"}, + names = {"--cps_mapped_java_publisher_count"}, description = "Number of cps mapped publishers to start." ) - private int cpsMappedPublisherCount = 0; + private int cpsMappedJavaPublisherCount = 0; @Parameter( names = {"--message_size", "-m"}, @@ -379,9 +379,10 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } - if (cpsMappedPublisherCount > 0) { + if (cpsMappedJavaPublisherCount > 0) { clientParamsMap.put( - new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), cpsMappedPublisherCount); + new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), + cpsMappedJavaPublisherCount); } if (kafkaPublisherCount > 0) { clientParamsMap.put( @@ -413,7 +414,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedJavaPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( From 4a579caff5c9e18cdd930694ef6cf61135618ab5 Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 036/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From f712a935ff4bc8389e4a8fad10ca8a8b4520aa96 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 037/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From 899e9b63efb91010c37dfd00b55702cc42916c66 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 038/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From bb1e3872d7e8ffbb8b6d876c6131e9c730242632 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 039/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From de07c8ba5721b76272cf6b1e2c6dbb78c62c2135 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 040/140] Added PubsubConsumer class --- load-test-framework/flic.iml | 117 --- .../clients/consumer/PubsubConsumer.java | 743 +++++++++++++++--- 2 files changed, 647 insertions(+), 213 deletions(-) delete mode 100644 load-test-framework/flic.iml diff --git a/load-test-framework/flic.iml b/load-test-framework/flic.iml deleted file mode 100644 index 6826b98d..00000000 --- a/load-test-framework/flic.iml +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 42b3f7db..12bb1ec5 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -10,103 +10,654 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; +package com.google.kafka.cients.consumer; public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; // this might need to be an /internal class - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - // currentThread holds the threadId of the current thread accessing PubsubConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file From 01cfedf74ad2b7a30f90b0ac5d461af4345f404f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 041/140] New maven setup for mapped api --- pubsub-mapped-api/pom.xml | 49 +++++++ .../clients/consumer/PubsubConsumer.java | 136 +++++++++++++++++- .../clients/producer/PubsubProducer.java | 2 +- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 0 .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 10 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 pubsub-mapped-api/pom.xml rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/consumer/PubsubConsumer.java (83%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/PubsubProducer.java (99%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/PubsubProducerTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml new file mode 100644 index 00000000..1bb14363 --- /dev/null +++ b/pubsub-mapped-api/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + com.google.pubsub + pubsub-mapped-api + jar + 1.0-SNAPSHOT + MappedApi + http://maven.apache.org + + + junit + junit + 4.12 + + + org.apache.kafka + kafka_2.10 + 0.10.0.0 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.slf4j + slf4j-api + 1.7.21 + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java similarity index 83% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 12bb1ec5..cadfbf1b 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -10,7 +10,64 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ +<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; +======= + +package com.google.kafka.clients.consumer; + +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.network.ChannelBuilder; +import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -105,7 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { + /* try { log.debug("Starting the Kafka consumer"); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); @@ -144,9 +201,82 @@ private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, D this.keyDeserializer = keyDeserializer; } if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; } - } // left off at kafka's line 645 + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ } /** diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java similarity index 99% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 2077a3c1..5f7d3507 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -85,7 +85,7 @@ public class PubsubProducer implements Producer { private final ProducerConfig producerConfig; private final long maxBlockTimeMs; private final int requestTimeoutMs; - private final ProducerInterceptors interceptors + private final ProducerInterceptors interceptors; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 3d192415582ca26d9405a146fdb70a24e70d6669 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 042/140] scratching some of the mapped api to make it more pub/sub --- pubsub-mapped-api/pom.xml | 20 + .../clients/consumer/PubsubConsumer.java | 119 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 6 files changed, 24 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1bb14363..ef3ab09f 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -33,6 +33,26 @@ slf4j-log4j12 1.7.21 + + io.grpc + grpc-all + 1.0.1 + + + io.grpc + grpc-netty + 1.0.1 + + + io.grpc + grpc-protobuf + 1.0.1 + + + io.grpc + grpc-stub + 1.0.1 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index cadfbf1b..72eb4413 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -14,7 +14,7 @@ package com.google.kafka.cients.consumer; ======= -package com.google.kafka.clients.consumer; +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -162,121 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -329,7 +215,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From ff248df0898810c7458cbe9ca70aad07adac91cd Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 043/140] Implementing publisher/producer using grpc backend. --- pubsub-mapped-api/pom.xml | 64 +- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../producer/PubsubProducerConfig.java | 85 ++ .../com/google/pubsub/common/PubsubUtils.java | 46 + .../src/main/resources/log4j.properties | 7 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 11 files changed, 1061 insertions(+), 1852 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java create mode 100644 pubsub-mapped-api/src/main/resources/log4j.properties delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index ef3ab09f..916913ca 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,6 +8,11 @@ MappedApi http://maven.apache.org + + com.google.pubsub + cloud-pubsub-client + 0.2-EXPERIMENTAL + junit junit @@ -16,7 +21,7 @@ org.apache.kafka kafka_2.10 - 0.10.0.0 + 0.10.1.1 org.apache.commons @@ -33,6 +38,11 @@ slf4j-log4j12 1.7.21 + + log4j + log4j + 1.2.17 + io.grpc grpc-all @@ -55,10 +65,62 @@ + + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + package + + shade + + + false + + + + com.google.pubsub.clients.ClientMain + + + mapped-api + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + + com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:0.14.0:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto/ + + + + + compile + compile-custom + + + + org.apache.maven.plugins maven-compiler-plugin + 3.6.1 1.8 1.8 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 72eb4413..e5abef92 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,14 +1,16 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ <<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; @@ -16,18 +18,17 @@ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -36,7 +37,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -71,608 +71,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java new file mode 100644 index 00000000..aa812494 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients.producer; + +import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.serialization.Serializer; + +public class PubsubProducerConfig extends AbstractConfig { + private static final ConfigDef CONFIG; + + public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + + public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + + public static final String BATCH_SIZE_CONFIG = "batch.size"; + private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; + + public static final String ACKS_CONFIG = "acks"; + private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project to use with the publisher."; + + static { + CONFIG = + new ConfigDef() + .define( + KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) + .define( + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + } + + PubsubProducerConfig(Map properties) { + super(CONFIG, properties); + } + + public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + newConfigs.putAll(configs); + if (keySerializer != null) { + newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newConfigs.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newConfigs; + } + + public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + Properties newProperties = new Properties(); + if (keySerializer != null) { + newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newProperties.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newProperties; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/resources/log4j.properties b/pubsub-mapped-api/src/main/resources/log4j.properties new file mode 100644 index 00000000..ff0cb5bd --- /dev/null +++ b/pubsub-mapped-api/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=DEBUG, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + +log4j.org.apache.kafka = OFF +log4j.logger.io.grpc = OFF +log4j.logger.io.netty = OFF \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From b82311347b024def6d34fe94899017ada15a3db7 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 044/140] Add details to simplified producer --- pubsub-mapped-api/pom.xml | 5 ++ .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../producer/PubsubProducerConfig.java | 6 +- .../internals/PubsubFutureRecordMetadata.java | 38 +++++++++++++ 4 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 916913ca..38b17362 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.cloud + google-cloud-pubsub + 0.9.2-alpha + junit junit diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index aa812494..5cee6425 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -43,6 +43,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; + private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + static { CONFIG = new ConfigDef() @@ -52,7 +55,8 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java new file mode 100644 index 00000000..88a5fd90 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +/*package com.google.pubsub.clients.producer.internals; + +private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; + } + + public boolean isCancelled() { + return false; + } + + public boolean isDone() { + return false; + } + + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; + } + + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +}*/ \ No newline at end of file From 1c9b7e717043f8946d2b603f2f8e7fc495033237 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 045/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 12 +++ .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 56 +++++++++++ .../pubsub/clients/ProducerThreadPool.java | 72 ++++++++++++++ .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 12 ++- ...ubsubUtils.java => PubsubChannelUtil.java} | 35 +++++-- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 8 files changed, 228 insertions(+), 178 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename pubsub-mapped-api/src/main/java/com/google/pubsub/common/{PubsubUtils.java => PubsubChannelUtil.java} (66%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 38b17362..99f6ff22 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,6 +23,18 @@ junit 4.12 + + org.powermock + powermock-module-junit4-legacy + 1.7.0RC2 + test + + + org.powermock + powermock-api-easymock + 1.7.0RC2 + test + org.apache.kafka kafka_2.10 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java new file mode 100644 index 00000000..092ac653 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -0,0 +1,56 @@ +package com.google.pubsub.clients; + +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; + + +public class ProducerThread implements Runnable { + private String command; + private PubsubProducer producer; + private String topic; + private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + + public ProducerThread(String s, Properties props, String topic) { + this.command = s; + this.producer = new PubsubProducer<>(props); + this.topic = topic; + } + + public void run() { + log.info("Start running the command"); + processCommand(); + log.info("End running the command"); + } + + private void processCommand() { + try { + ProducerRecord msg = new ProducerRecord(topic, "message" + command); + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } + } + ); + Thread.sleep(5000); + producer.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public String toString() { + return command; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java new file mode 100644 index 00000000..b37cba9e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -0,0 +1,72 @@ +package com.google.pubsub.clients; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class ProducerThreadPool { + private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); + + public static void main(String[] args) { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); + threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread t, Throwable e) { + log.error(t + " throws exception: " + e); + } + }); + + //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); + ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); + + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + + /* props.putAll(new ImmutableMap.Builder<>() + .put("max.block.ms", "30000") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("bootstrap.servers", "104.198.72.101:9092") + .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB + // 10M, high enough to allow for duration to control batching + .put("batch.size", Integer.toString(10 * 1000 * 1000)) + .put("linger.ms", 10) + .build() + );*/ + + for (int i = 0; i < 20; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0]); + executor.execute(worker); + } + + executor.shutdown(); + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + log.error("Executor did not terminate"); + } + } + } catch (InterruptedException ie) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..bcf86059 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null()); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 88a5fd90..771fcb82 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -13,9 +13,15 @@ * the License. */ -/*package com.google.pubsub.clients.producer.internals; +package com.google.pubsub.clients.producer.internals; -private static class PubsubFutureRecordMetadata implements Future { +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.producer.RecordMetadata; + +public class PubsubFutureRecordMetadata implements Future { public boolean cancel(boolean b) { return false; } @@ -35,4 +41,4 @@ public RecordMetadata get() throws InterruptedException, ExecutionException { public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } -}*/ \ No newline at end of file +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 10a5599e..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,31 +16,46 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; import io.grpc.auth.ClientAuthInterceptor; import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.concurrent.Executors; -public class PubsubUtils { +public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); + + private ManagedChannel channel; + private CallCredentials callCredentials; + + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + callCredentials = MoreCallCredentials.from(credentials); + channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + } + + public CallCredentials callCredentials() { + return callCredentials; + } + + public Channel channel() { + return channel; + } + + public void closeChannel() { + channel.shutdown(); } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From b530150db24ae8035c7d380d8a345e16550bf9e5 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 046/140] Working on unit tests, need to mock the publisher calls --- pubsub-mapped-api/pom.xml | 17 ++--- .../google/pubsub/clients/ProducerThread.java | 26 ++++---- .../pubsub/clients/ProducerThreadPool.java | 10 +-- .../clients/producer/PubsubProducer.java | 30 ++------- .../producer/PubsubProducerConfig.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- .../pubsub/common/PubsubChannelUtilTest.java | 51 +++++++++++++++ 8 files changed, 134 insertions(+), 77 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 99f6ff22..1fdd87b7 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,18 +23,6 @@ junit 4.12 - - org.powermock - powermock-module-junit4-legacy - 1.7.0RC2 - test - - - org.powermock - powermock-api-easymock - 1.7.0RC2 - test - org.apache.kafka kafka_2.10 @@ -80,6 +68,11 @@ grpc-stub 1.0.1 + + org.mockito + mockito-all + 2.0.2-beta + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 092ac653..e075ae36 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -30,19 +30,21 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord(topic, "message" + command); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } } - } - } - ); + ); + } Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index b37cba9e..4b168cc0 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -33,15 +33,15 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 1) + .put("batch.size", 20) .put("linger.ms", 1) .build() ); /* props.putAll(new ImmutableMap.Builder<>() .put("max.block.ms", "30000") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") .put("bootstrap.servers", "104.198.72.101:9092") .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB @@ -49,9 +49,9 @@ public void uncaughtException(Thread t, Throwable e) { .put("batch.size", Integer.toString(10 * 1000 * 1000)) .put("linger.ms", 10) .build() - );*/ + ); */ - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 1; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index bcf86059..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -273,7 +251,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null()); + doSend(request, null, null); } } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 5cee6425..ff3e6139 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -76,8 +76,8 @@ public static Map addSerializerToConfig(Map conf } public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap(); Properties newProperties = new Properties(); + newProperties.putAll(properties); if (keySerializer != null) { newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java new file mode 100644 index 00000000..2c5ad730 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.common; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PubsubChannelUtilTest { + + private static PubsubChannelUtil channelUtil; + + @BeforeClass + public static void setUp() throws IOException { + channelUtil = new PubsubChannelUtil(); + } + + @AfterClass + public static void tearDown() { + channelUtil.closeChannel(); + } + + @Test + public void testGetCallCredentials() throws IOException { + assertNotNull(channelUtil.callCredentials()); + channelUtil.closeChannel(); + } + + @Test + public void testGetChannel() { + assertNotNull(channelUtil.channel()); + } +} From 6239c08550dfec7ab78e9a51eb2f13d719887ab6 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 047/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 6 +- .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../producer/PubsubProducerConfig.java | 10 +- .../pubsub/common/PubsubChannelUtil.java | 4 - .../producer/PubsubProducerConfigTest.java | 59 +++++++++++ .../clients/producer/PubsubProducerTest.java | 35 +----- 6 files changed, 166 insertions(+), 48 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index e075ae36..2a3bb585 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -2,6 +2,7 @@ import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; +import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; @@ -18,7 +19,10 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic) { this.command = s; - this.producer = new PubsubProducer<>(props); + //this.producer = new PubsubProducer<>(props); + this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) + .batchSize(props.getProperty("batch.size")) + .) this.topic = topic; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index ff3e6139..f03c017f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -46,6 +46,10 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final boolean DEFAULT_ACKS = true; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + static { CONFIG = new ConfigDef() @@ -53,10 +57,10 @@ public class PubsubProducerConfig extends AbstractConfig { KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java new file mode 100644 index 00000000..000536cd --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -0,0 +1,59 @@ +package com.google.pubsub.clients.producer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Properties; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.serialization.Serializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + + +public class PubsubProducerConfigTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testSuccessAllConfigsProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "unit-test-project") + .build() + ); + + PubsubProducerConfig testConfig = new PubsubProducerConfig(props); + + assertEquals("Project config equals unit-test-project.", "unit-test-project", testConfig.getString(PubsubProducerConfig.PROJECT_CONFIG)); + assertNotNull("Key serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + assertNotNull("Value serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + } + + @Test + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "unit-test-project"); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + + } + + @Test + public void testNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From f7483380c01aa7ce5efa3dcc53b0c18680390d8a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 048/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 16 +++++++++------- .../pubsub/clients/ProducerThreadPool.java | 7 ++++--- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 2a3bb585..d1ae43fa 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,6 +1,8 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; @@ -17,12 +19,12 @@ public class ProducerThread implements Runnable { private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); - public ProducerThread(String s, Properties props, String topic) { + public ProducerThread(String s, Properties props, String topic) throws IOException { this.command = s; - //this.producer = new PubsubProducer<>(props); - this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) - .batchSize(props.getProperty("batch.size")) - .) + this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + .batchSize(Integer.parseInt(props.getProperty("batch.size"))) + .isAcks(props.getProperty("acks").matches("1|all")) + .build(); this.topic = topic; } @@ -34,8 +36,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 4b168cc0..edb9707b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -1,5 +1,6 @@ package com.google.pubsub.clients; +import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -15,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -33,8 +34,8 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 20) - .put("linger.ms", 1) + .put("batch.size", "1") + .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From ca6419ba8aa35a33c0bb83a25f5d6d1b4757c78a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:46:08 -0800 Subject: [PATCH 049/140] Minor fixes to producer's classes --- .../pubsub/clients/consumer/PubsubConsumer.java | 3 --- .../pubsub/clients/producer/PubsubProducer.java | 2 +- .../com/google/pubsub/common/PubsubChannelUtil.java | 13 +++++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index e5abef92..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -12,9 +12,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ -<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java -package com.google.kafka.cients.consumer; -======= package com.google.pubsub.clients.consumer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..e9ca5753 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -360,7 +360,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..9088d5bf 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +44,14 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } From 02bfc7c6d92763c40e004ad10ca08fbb36914e5f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 10:06:42 -0800 Subject: [PATCH 050/140] Omitting unit tests and unused classes --- .../internals/PubsubFutureRecordMetadata.java | 44 -------------- .../clients/producer/PubsubProducerTest.java | 60 ------------------- .../pubsub/common/PubsubChannelUtilTest.java | 51 ---------------- 3 files changed, 155 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java deleted file mode 100644 index 771fcb82..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.pubsub.clients.producer.internals; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.kafka.clients.producer.RecordMetadata; - -public class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index 32555be2..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - - /* send() tests */ - @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub - - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal - } - - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); - - return new PubsubProducer(props); - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java deleted file mode 100644 index 2c5ad730..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.pubsub.common; - -import static org.junit.Assert.assertNotNull; - -import java.io.IOException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PubsubChannelUtilTest { - - private static PubsubChannelUtil channelUtil; - - @BeforeClass - public static void setUp() throws IOException { - channelUtil = new PubsubChannelUtil(); - } - - @AfterClass - public static void tearDown() { - channelUtil.closeChannel(); - } - - @Test - public void testGetCallCredentials() throws IOException { - assertNotNull(channelUtil.callCredentials()); - channelUtil.closeChannel(); - } - - @Test - public void testGetChannel() { - assertNotNull(channelUtil.channel()); - } -} From 8331f4d51f6347a21814d1d6d48394bcbb631b1a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 11:06:38 -0800 Subject: [PATCH 051/140] Eliminated deprecated client use in pom; added to .travis.yml --- .travis.yml | 1 + pubsub-mapped-api/pom.xml | 27 +------------------ .../clients/producer/PubsubProducer.java | 2 -- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70f2177d..fb2a5a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: script: - mvn -q -B -f jms-light/pom.xml clean verify - mvn -q -B -f kafka-connector/pom.xml clean verify + - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 - mvn -q -B -f load-test-framework/pom.xml clean verify diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1fdd87b7..976ad2f8 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,15 +8,10 @@ MappedApi http://maven.apache.org - - com.google.pubsub - cloud-pubsub-client - 0.2-EXPERIMENTAL - com.google.cloud google-cloud-pubsub - 0.9.2-alpha + 0.9.4-alpha junit @@ -48,26 +43,6 @@ log4j 1.2.17 - - io.grpc - grpc-all - 1.0.1 - - - io.grpc - grpc-netty - 1.0.1 - - - io.grpc - grpc-protobuf - 1.0.1 - - - io.grpc - grpc-stub - 1.0.1 - org.mockito mockito-all diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index e9ca5753..83550495 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,7 +16,6 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -27,7 +26,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; From fb252a2c8ebdeef7a411f17b99ba51077574dc2c Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 11:59:54 -0800 Subject: [PATCH 052/140] Modified .travis.yml to run mapped_api with jdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fb2a5a34..8a4957b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: script: - mvn -q -B -f jms-light/pom.xml clean verify - mvn -q -B -f kafka-connector/pom.xml clean verify - - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 + - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f load-test-framework/pom.xml clean verify From 3acb58ee1ac8a04367e814a0443910a3967a87e5 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Mar 2017 15:39:22 -0800 Subject: [PATCH 053/140] Fixed requested changes on style, naming, and licenses. --- .../clients/consumer/PubsubConsumer.java | 429 +----------------- .../clients/producer/PubsubProducer.java | 159 ++++--- .../producer/PubsubProducerConfig.java | 57 ++- .../clients/{ => tests}/ProducerThread.java | 31 +- .../{ => tests}/ProducerThreadPool.java | 60 ++- .../pubsub/common/PubsubChannelUtil.java | 29 +- 6 files changed, 207 insertions(+), 558 deletions(-) rename pubsub-mapped-api/src/main/java/com/google/pubsub/clients/{ => tests}/ProducerThread.java (66%) rename pubsub-mapped-api/src/main/java/com/google/pubsub/clients/{ => tests}/ProducerThreadPool.java (54%) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 285c5d8e..5e8fd2d8 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,69 +1,39 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.clients.consumer; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.common.Cluster; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.network.ChannelBuilder; -import org.apache.kafka.common.network.Selector; -import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetSocketAddress; import java.util.Collection; -import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -88,502 +58,133 @@ public PubsubConsumer(Properties properties, } - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ public Set assignment() { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ public Set subscription() { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ public void subscribe(Collection topics, ConsumerRebalanceListener listener) { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ @Override public void subscribe(Collection topics) { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ @Override public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { throw new NotImplementedException("Not yet implemented"); } - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ public void unsubscribe() { throw new NotImplementedException("Not yet implemented"); } - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ @Override public void assign(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ @Override public ConsumerRecords poll(long timeout) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ @Override public void commitSync() { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ @Override public void commitSync(final Map offsets) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ @Override public void commitAsync() { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ @Override public void commitAsync(OffsetCommitCallback callback) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ @Override public void commitAsync(final Map offsets, OffsetCommitCallback callback) { throw new NotImplementedException("Not yet implemented"); } - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ @Override public void seek(TopicPartition partition, long offset) { throw new NotImplementedException("Not yet implemented"); } - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ public void seekToBeginning(Collection partitions) { } - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ public void seekToEnd(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public long position(TopicPartition partition) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public OffsetAndMetadata committed(TopicPartition partition) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the metrics kept by the consumer - */ public Map metrics() { throw new NotImplementedException("Not yet implemented"); } - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public List partitionsFor(String topic) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public Map> listTopics() { throw new NotImplementedException("Not yet implemented"); } - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ public void pause(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ public void resume(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ public Set paused() { throw new NotImplementedException("Not yet implemented"); } - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ public Map offsetsForTimes(Map timestampsToSearch) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ public Map beginningOffsets(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ public Map endOffsets(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ public void close() { } - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ public void close(long timeout, TimeUnit timeUnit) { throw new NotImplementedException("Not yet implemented"); } - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ public void wakeup() { throw new NotImplementedException("Not yet implemented"); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 83550495..1a573612 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,17 +1,18 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.clients.producer; @@ -97,7 +98,8 @@ public PubsubProducer(Map configs) { public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + this(new PubsubProducerConfig( + PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } @@ -107,16 +109,19 @@ public PubsubProducer(Properties properties) { public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + this(new PubsubProducerConfig( + PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, + Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) + .withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -150,17 +155,14 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } /** - * Send the given record asynchronously and return a future which will eventually contain the response information. - * - * @param record The record to send - * @return A future which will eventually contain the response information + * Sends the given record. */ public Future send(ProducerRecord record) { return send(record, null); } /** - * Send a record and invoke the given callback when the record has been acknowledged by the server + * Sends the given record and invokes the specified callback. */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); @@ -174,7 +176,8 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes + .put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -190,9 +193,9 @@ public Future send(ProducerRecord record, Callback callbac PubsubMessage message = PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(valueBytes)) - .putAllAttributes(attributes) - .build(); + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); @@ -203,20 +206,22 @@ public Future send(ProducerRecord record, Callback callbac long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); RecordAccumulator.RecordAppendResult result = new RecordAppendResult( new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, + false); if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); doSend(request, callback, result); } return result.future; } - private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { + private Future doSend(PublishRequest request, Callback callback, + RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -243,58 +248,58 @@ public void onFailure(Throwable t) { perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { - return new FutureFailure(e); + log.error("Exception occurred during send: " + e); + return null; } return result.future; } private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" - + " configured"); + throw new RecordTooLargeException( + "Message is " + size + " bytes which is larger than max request size you have" + + " configured"); } } /** - * Flush any accumulated records from the producer. Blocks until all sends are complete. + * Flushes records that have accumulated. */ public void flush() { log.debug("Flushing..."); for (String topic : perTopicBatches.keySet()) { PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); doSend(request, null, null); } } /** - * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change - * over time so this list should not be cached. + * Not supported by this implementation. */ public List partitionsFor(String topic) { throw new NotImplementedException("Partitions not supported"); } /** - * Return a map of metrics maintained by the producer + * Not supported by this implementation. */ public Map metrics() { throw new NotImplementedException("Metrics not supported."); } /** - * Close this producer + * Closes the producer. */ public void close() { close(0, null); } /** - * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the - * timeout, fail any pending send requests and force close the producer. + * Closes the producer with the given timeout. */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { @@ -306,7 +311,12 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + /** + * PubsubProducer.Builder is used to create an instance of the publisher, with the specified + * properties and configurations. + */ public static class Builder { + private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; @@ -320,7 +330,8 @@ public static class Builder { private Time time; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { - Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); + Preconditions + .checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; @@ -336,7 +347,10 @@ private void setDefaults() { this.time = new SystemTime(); } - public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + public Builder publisherFutureStub(PublisherFutureStub val) { + publisher = val; + return this; + } public Builder batchSize(int val) { Preconditions.checkArgument(val > 0); @@ -344,9 +358,15 @@ public Builder batchSize(int val) { return this; } - public Builder isAcks(boolean val) { isAcks = val; return this; } + public Builder isAcks(boolean val) { + isAcks = val; + return this; + } - public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + public Builder perTopicBatches(Map> val) { + perTopicBatches = val; + return this; + } public Builder maxRequestSize(int val) { Preconditions.checkArgument(val >= 0); @@ -354,9 +374,15 @@ public Builder maxRequestSize(int val) { return this; } - public Builder time(Time val) { time = val; return this; } + public Builder time(Time val) { + time = val; + return this; + } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + public Builder pubsubChannelUtil(PubsubChannelUtil val) { + channelUtil = val; + return this; + } public PubsubProducer build() { // this is where to set fields w/ side effects @@ -364,38 +390,11 @@ public PubsubProducer build() { this.channelUtil = new PubsubChannelUtil(); } if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) + .withCallCredentials(channelUtil.callCredentials()); } return new PubsubProducer(this); } } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ - private static class FutureFailure implements Future { - private final ExecutionException exception; - - public FutureFailure(Exception e) { - this.exception = new ExecutionException(e); - } - - public boolean cancel(boolean interrupt) { - return false; - } - - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } - - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return true; - } - } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index f03c017f..cbaed929 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -1,17 +1,19 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + package com.google.pubsub.clients.producer; import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; @@ -25,20 +27,26 @@ import org.apache.kafka.common.config.ConfigDef.Type; import org.apache.kafka.common.serialization.Serializer; +/** + * Provides the configurations for a PubsubProducer instance. + */ public class PubsubProducerConfig extends AbstractConfig { private static final ConfigDef CONFIG; public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; - private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements " + + "the Serializer interface."; public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; - private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that " + + "implements the Serializer interface."; public static final String BATCH_SIZE_CONFIG = "batch.size"; private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; public static final String ACKS_CONFIG = "acks"; - private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + private static final String ACKS_DOC = "Whether server acks are needed before a publish " + + "request completes."; public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; @@ -56,18 +64,22 @@ public class PubsubProducerConfig extends AbstractConfig { .define( KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( - VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, + VALUE_SERIALIZER_CLASS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, + BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), + Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { super(CONFIG, properties); } - public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + public static Map addSerializerToConfig(Map configs, + Serializer keySerializer, Serializer valueSerializer) { Map newConfigs = new HashMap(); newConfigs.putAll(configs); if (keySerializer != null) { @@ -79,7 +91,8 @@ public static Map addSerializerToConfig(Map conf return newConfigs; } - public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + public static Properties addSerializerToConfig(Properties properties, + Serializer keySerializer, Serializer valueSerializer) { Properties newProperties = new Properties(); newProperties.putAll(properties); if (keySerializer != null) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index d1ae43fa..fc02a198 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -1,31 +1,50 @@ -package com.google.pubsub.clients; +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.tests; import com.google.pubsub.clients.producer.PubsubProducer; import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; - +/** + * Creates a thread that sends a given series of messages. Serves as a sanity check for the + * PubsubProducer implementation. + */ public class ProducerThread implements Runnable { private String command; private PubsubProducer producer; private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + private int numMessages; - public ProducerThread(String s, Properties props, String topic) throws IOException { + public ProducerThread(String s, Properties props, String topic, int numMessages) { this.command = s; this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .build(); this.topic = topic; + this.numMessages = numMessages; } public void run() { @@ -37,7 +56,7 @@ public void run() { private void processCommand() { try { ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); - for (int i = 0; i < 1; i++) { + for (int i = 0; i < numMessages; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java similarity index 54% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index edb9707b..820f889f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -1,22 +1,43 @@ -package com.google.pubsub.clients; +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// -import java.io.IOException; +package com.google.pubsub.clients.tests; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; +/** + * Example main class to start up several ProducerThreads with a specified topic. + * Number of threads will also be specified by a command-line argument. + * arg[0] = topic, arg[1] = number of threads, arg[2] = number of messages to send. + * Topic is required; if number of threads is not specified, 2 is the default. + * If number of messages is not specified, 1 is the default. + */ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) throws IOException { + public static void main(String[] args) { + Preconditions.checkArgument(args[0] != null); ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -25,7 +46,6 @@ public void uncaughtException(Thread t, Throwable e) { } }); - //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); Properties props = new Properties(); @@ -39,21 +59,17 @@ public void uncaughtException(Thread t, Throwable e) { .build() ); - /* props.putAll(new ImmutableMap.Builder<>() - .put("max.block.ms", "30000") - //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("bootstrap.servers", "104.198.72.101:9092") - .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB - // 10M, high enough to allow for duration to control batching - .put("batch.size", Integer.toString(10 * 1000 * 1000)) - .put("linger.ms", 10) - .build() - ); */ + int numThreads = 2; + if (args[1] != null) { + numThreads = Integer.parseInt(args[1]); + } + int numMessages = 1; + if (args[2] != null) { + numMessages = Integer.parseInt(args[2]); + } - for (int i = 0; i < 1; i++) { - Runnable worker = new ProducerThread("" + i, props, args[0]); + for (int i = 0; i < numThreads; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0], numMessages); executor.execute(worker); } @@ -70,4 +86,4 @@ public void uncaughtException(Thread t, Throwable e) { Thread.currentThread().interrupt(); } } -} \ No newline at end of file +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 9088d5bf..5bc9f9b1 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -1,17 +1,18 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.common; From 263a64a41f4008757183fc668177b555cc722b55 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Mar 2017 10:57:32 -0800 Subject: [PATCH 054/140] Small fixes to concurrency issues in Producer. --- .../clients/producer/PubsubProducer.java | 49 ++++++------------- .../clients/tests/ProducerThreadPool.java | 6 +-- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 1a573612..daa8606b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PublishRequest; import com.google.pubsub.v1.PublishResponse; @@ -32,10 +33,6 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; -import org.apache.kafka.clients.producer.internals.ProduceRequestResult; -import org.apache.kafka.clients.producer.internals.RecordAccumulator; -import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -43,8 +40,6 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.SystemTime; -import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,8 +69,8 @@ public class PubsubProducer implements Producer { private final boolean isAcks; private final Map> perTopicBatches; private final int maxRequestSize; - private final Time time; private final PubsubChannelUtil channelUtil; + private final SettableFuture future; private boolean closed = false; @@ -86,10 +81,10 @@ private PubsubProducer(Builder builder) { isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; - time = builder.time; channelUtil = builder.channelUtil; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; + future = SettableFuture.create(); } public PubsubProducer(Map configs) { @@ -118,7 +113,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) .withCallCredentials(channelUtil.callCredentials()); @@ -144,12 +138,12 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - + future = SettableFuture.create(); batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + perTopicBatches = new HashMap<>(); log.debug("Producer successfully initialized."); } @@ -177,7 +171,7 @@ public Future send(ProducerRecord record, Callback callbac if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); attributes - .put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -203,25 +197,19 @@ public Future send(ProducerRecord record, Callback callbac } batch.add(message); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - RecordAccumulator.RecordAppendResult result = new RecordAppendResult( - new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, - false); - if (result.batchIsFull) { + if (batch.size() == batchSize) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(PubsubChannelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback, result); + doSend(request, callback); } - return result.future; + return future; } - private Future doSend(PublishRequest request, Callback callback, - RecordAppendResult result) { + private Future doSend(PublishRequest request, Callback callback) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -248,10 +236,10 @@ public void onFailure(Throwable t) { perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { - log.error("Exception occurred during send: " + e); - return null; + future.setException(e); + // TODO: fix this so it's not unused } - return result.future; + return future; } private void checkRecordSize(int size) { @@ -273,7 +261,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null, null); + doSend(request, null); } } @@ -327,7 +315,6 @@ public static class Builder { private boolean isAcks; private Map> perTopicBatches; private int maxRequestSize; - private Time time; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { Preconditions @@ -344,7 +331,6 @@ private void setDefaults() { this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; - this.time = new SystemTime(); } public Builder publisherFutureStub(PublisherFutureStub val) { @@ -374,11 +360,6 @@ public Builder maxRequestSize(int val) { return this; } - public Builder time(Time val) { - time = val; - return this; - } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 820f889f..983e0118 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -37,7 +37,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); public static void main(String[] args) { - Preconditions.checkArgument(args[0] != null); + Preconditions.checkArgument(args.length > 0); ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -60,11 +60,11 @@ public void uncaughtException(Thread t, Throwable e) { ); int numThreads = 2; - if (args[1] != null) { + if (args.length > 1) { numThreads = Integer.parseInt(args[1]); } int numMessages = 1; - if (args[2] != null) { + if (args.length > 2) { numMessages = Integer.parseInt(args[2]); } From b08fc94e63f20bac98a3c8d0ff17a07362c5a1f2 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 13 Mar 2017 16:49:53 -0700 Subject: [PATCH 055/140] Implementing producer using CPS Publisher API. --- .../clients/producer/PubsubProducer.java | 174 +++++++----------- .../clients/tests/ProducerThreadPool.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 2 - 3 files changed, 68 insertions(+), 110 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index daa8606b..3b4c4961 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -17,17 +17,21 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.api.gax.core.RpcFuture; +import com.google.api.gax.core.RpcFutureCallback; +import com.google.api.gax.grpc.BundlingSettings; +import com.google.api.gax.grpc.FlowControlSettings; +import com.google.cloud.pubsub.spi.v1.Publisher; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -40,12 +44,11 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; +import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,30 +64,24 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private final PublisherFutureStub publisher; private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; private final int batchSize; private final boolean isAcks; - private final Map> perTopicBatches; private final int maxRequestSize; - private final PubsubChannelUtil channelUtil; - private final SettableFuture future; + private final Map publishers; private boolean closed = false; private PubsubProducer(Builder builder) { - publisher = builder.publisher; project = builder.project; batchSize = builder.batchSize; isAcks = builder.isAcks; - perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; - channelUtil = builder.channelUtil; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; - future = SettableFuture.create(); + publishers = new HashMap<>(); } public PubsubProducer(Map configs) { @@ -113,9 +110,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) - .withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -138,12 +132,11 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - future = SettableFuture.create(); batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = new HashMap<>(); + publishers = new HashMap<>(); log.debug("Producer successfully initialized."); } @@ -164,81 +157,71 @@ public Future send(ProducerRecord record, Callback callbac throw new RuntimeException("Publisher is closed"); } - String topic = record.topic(); + TopicName topic = TopicName.create(project, record.topic()); Map attributes = new HashMap<>(); byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic.getTopic(), record.key()); attributes .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - if (project == null) { - throw new RuntimeException("No project specified."); - } - byte[] valueBytes = ByteString.EMPTY.toByteArray(); if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic, record.value()); + valueBytes = valueSerializer.serialize(topic.getTopic(), record.value()); } checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + synchronized (publishers) { + if (!publishers.containsKey(topic)) { + try { + Publisher newPub = Publisher.newBuilder(topic) + .setBundlingSettings(BundlingSettings.newBuilder() + .setIsEnabled(true) + .setElementCountThreshold((long) batchSize) + .setDelayThreshold(new Duration(1)) + .setRequestByteThreshold(1000L) + .build()) + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(maxRequestSize) + .build()) + .build(); + publishers.put(topic, newPub); + } catch (IOException e) { + log.error("Exception occurred: " + e); + } + } + } + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatches.get(topic); - if (batch == null) { - batch = new ArrayList<>(batchSize); - perTopicBatches.put(topic, batch); - } - batch.add(message); - - if (batch.size() == batchSize) { - log.trace("Sending a batch of messages."); - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(PubsubChannelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); - doSend(request, callback); - } - return future; - } - private Future doSend(PublishRequest request, Callback callback) { - try { - ListenableFuture response = publisher.publish(request); - if (callback != null) { - if (isAcks) { - Futures.addCallback( - response, - new FutureCallback() { - public void onSuccess(PublishResponse response) { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } - - public void onFailure(Throwable t) { - callback.onCompletion(null, new Exception(t)); - } - } - ); - } else { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } + RpcFuture messageIdFuture = publishers.get(topic).publish(message); + Future future = SettableFuture.create(); + + if (callback != null) { + if (isAcks) { + messageIdFuture.addCallback(new RpcFutureCallback() { + @Override + public void onFailure(Throwable t) { + callback.onCompletion(null, new ExecutionException(t)); + } + + @Override + public void onSuccess(String result) { + callback.onCompletion(null, null); + } + }); } else { - response.get(); - perTopicBatches.clear(); + callback.onCompletion(null, null); } - } catch (InterruptedException | ExecutionException e) { - future.setException(e); - // TODO: fix this so it's not unused } + return future; } @@ -255,13 +238,12 @@ private void checkRecordSize(int size) { */ public void flush() { log.debug("Flushing..."); - for (String topic : perTopicBatches.keySet()) { - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); - doSend(request, null); + for (TopicName topic : publishers.keySet()) { + try { + publishers.get(topic).shutdown(); + } catch (Exception e) { + log.error("Exception occurred during flush: " + e); + } } } @@ -293,8 +275,13 @@ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { throw new IllegalArgumentException("Timeout cannot be negative."); } - - channelUtil.closeChannel(); + for (TopicName topic : publishers.keySet()) { + try { + publishers.get(topic).shutdown(); + } catch (Exception e) { + log.error("Exception occurred during close: " + e); + } + } log.debug("Closed producer"); closed = true; } @@ -309,11 +296,8 @@ public static class Builder { private final Serializer keySerializer; private final Serializer valueSerializer; - private PubsubChannelUtil channelUtil; - private PublisherFutureStub publisher; private int batchSize; private boolean isAcks; - private Map> perTopicBatches; private int maxRequestSize; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { @@ -329,15 +313,9 @@ private void setDefaults() { // this is where to set 'regular' fields w/o side effects this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; - this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; } - public Builder publisherFutureStub(PublisherFutureStub val) { - publisher = val; - return this; - } - public Builder batchSize(int val) { Preconditions.checkArgument(val > 0); batchSize = val; @@ -349,31 +327,13 @@ public Builder isAcks(boolean val) { return this; } - public Builder perTopicBatches(Map> val) { - perTopicBatches = val; - return this; - } - public Builder maxRequestSize(int val) { Preconditions.checkArgument(val >= 0); maxRequestSize = val; return this; } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { - channelUtil = val; - return this; - } - public PubsubProducer build() { - // this is where to set fields w/ side effects - if (channelUtil == null) { - this.channelUtil = new PubsubChannelUtil(); - } - if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) - .withCallCredentials(channelUtil.callCredentials()); - } return new PubsubProducer(this); } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 983e0118..94478810 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -54,7 +54,7 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", "1") + .put("batch.size", "5") .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 5bc9f9b1..062f4d57 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -17,8 +17,6 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; From 8c6ccc32f9fd0e861a27e76b27712421d820d58e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 10:10:58 -0700 Subject: [PATCH 056/140] Ensuring that the configs are consistent across types of publisher. --- .../java/com/google/pubsub/clients/mapped/CPSPublisherTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index e100f069..d8fa81b8 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -35,7 +35,7 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) - .batchSize(this.batchSize) + .batchSize(9500000) .isAcks(true) .build(); } From 43c25e51c86d22569be65ba2fccc5b03fc748844 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 11:29:38 -0700 Subject: [PATCH 057/140] Added more configs; added metadata to return for send() --- .../clients/producer/PubsubProducer.java | 52 +++++++++++++------ .../producer/PubsubProducerConfig.java | 14 ++++- .../pubsub/clients/tests/ProducerThread.java | 4 +- .../clients/tests/ProducerThreadPool.java | 2 +- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 3b4c4961..8b977abb 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -22,10 +22,6 @@ import com.google.api.gax.grpc.BundlingSettings; import com.google.api.gax.grpc.FlowControlSettings; import com.google.cloud.pubsub.spi.v1.Publisher; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; @@ -40,6 +36,7 @@ import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; @@ -63,13 +60,16 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 1000; private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; - private final int batchSize; + private final long batchSize; + private final int bufferMemory; private final boolean isAcks; private final int maxRequestSize; + private final long lingerMs; private final Map publishers; private boolean closed = false; @@ -81,6 +81,8 @@ private PubsubProducer(Builder builder) { maxRequestSize = builder.maxRequestSize; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; + lingerMs = builder.lingerMs; + bufferMemory = builder.bufferMemory; publishers = new HashMap<>(); } @@ -132,10 +134,12 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + batchSize = configs.getLong(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); + bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); log.debug("Producer successfully initialized."); @@ -180,12 +184,12 @@ public Future send(ProducerRecord record, Callback callbac Publisher newPub = Publisher.newBuilder(topic) .setBundlingSettings(BundlingSettings.newBuilder() .setIsEnabled(true) - .setElementCountThreshold((long) batchSize) - .setDelayThreshold(new Duration(1)) - .setRequestByteThreshold(1000L) + .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) + .setDelayThreshold(Duration.millis(lingerMs)) + .setRequestByteThreshold(batchSize) .build()) .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(maxRequestSize) + .setMaxOutstandingRequestBytes(bufferMemory) .build()) .build(); publishers.put(topic, newPub); @@ -203,22 +207,24 @@ public Future send(ProducerRecord record, Callback callbac RpcFuture messageIdFuture = publishers.get(topic).publish(message); Future future = SettableFuture.create(); + RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic.getTopic(), 0), + 0, 0, System.currentTimeMillis(), 0, serializedKey.length, valueBytes.length); if (callback != null) { if (isAcks) { messageIdFuture.addCallback(new RpcFutureCallback() { @Override public void onFailure(Throwable t) { - callback.onCompletion(null, new ExecutionException(t)); + callback.onCompletion(metadata, new ExecutionException(t)); } @Override public void onSuccess(String result) { - callback.onCompletion(null, null); + callback.onCompletion(metadata, null); } }); } else { - callback.onCompletion(null, null); + callback.onCompletion(metadata, null); } } @@ -296,7 +302,9 @@ public static class Builder { private final Serializer keySerializer; private final Serializer valueSerializer; - private int batchSize; + private long batchSize; + private long lingerMs; + private int bufferMemory; private boolean isAcks; private int maxRequestSize; @@ -314,9 +322,11 @@ private void setDefaults() { this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.lingerMs = PubsubProducerConfig.DEFAULT_LINGER_MS; + this.bufferMemory = PubsubProducerConfig.DEFAULT_BUFFER_MEMORY; } - public Builder batchSize(int val) { + public Builder batchSize(long val) { Preconditions.checkArgument(val > 0); batchSize = val; return this; @@ -333,6 +343,18 @@ public Builder maxRequestSize(int val) { return this; } + public Builder lingerMs(long val) { + Preconditions.checkArgument(val >= 0); + lingerMs = val; + return this; + } + + public Builder bufferMemory(int val) { + Preconditions.checkArgument(val >= 0); + bufferMemory = val; + return this; + } + public PubsubProducer build() { return new PubsubProducer(this); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index cbaed929..505fda20 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -38,7 +38,7 @@ public class PubsubProducerConfig extends AbstractConfig { + "the Serializer interface."; public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; - private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that " + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that " + "implements the Serializer interface."; public static final String BATCH_SIZE_CONFIG = "batch.size"; @@ -54,9 +54,17 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final String LINGER_MS_CONFIG = "linger.ms"; + private static final String LINGER_MS_DOC = "Referred to as delayThreshold in CPS."; + + public static final String BUFFER_MEMORY_CONFIG = "buffer.memory"; + private static final String BUFFER_MEMORY_DOC = "Referred to as maxOutstandingRequestBytes in CPS."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final long DEFAULT_LINGER_MS = 0L; public static final boolean DEFAULT_ACKS = true; - public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1024*1024; + public static final int DEFAULT_BUFFER_MEMORY = 32 * 1024 * 1024; static { CONFIG = @@ -66,9 +74,11 @@ public class PubsubProducerConfig extends AbstractConfig { .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BUFFER_MEMORY_CONFIG, Type.LONG, DEFAULT_BUFFER_MEMORY, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(LINGER_MS_CONFIG, Type.LONG, DEFAULT_LINGER_MS, atLeast(0L), Importance.MEDIUM, LINGER_MS_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index fc02a198..94fd05bd 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -42,6 +42,7 @@ public ProducerThread(String s, Properties props, String topic, int numMessages) this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) + .lingerMs(Long.parseLong(props.getProperty("linger.ms"))) .build(); this.topic = topic; this.numMessages = numMessages; @@ -55,8 +56,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); for (int i = 0; i < numMessages; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command + i); producer.send( msg, new Callback() { @@ -65,6 +66,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { log.error("Exception sending the message: " + exception.getMessage()); } else { log.info("Successfully sent message"); + log.info("HERE'S THE METADATA: " + metadata); } } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 94478810..f94b3cc3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -54,7 +54,7 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", "5") + .put("batch.size", "1000") .put("linger.ms", "1") .build() ); From f030cbb2b58a74f43854d946745461c35d71f42d Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 12:40:42 -0700 Subject: [PATCH 058/140] Adjusting settings on mapped/CPSPublisherTask. --- .../com/google/pubsub/clients/mapped/CPSPublisherTask.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index d8fa81b8..13a098fe 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -3,6 +3,7 @@ import com.beust.jcommander.JCommander; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.util.Durations; import com.google.pubsub.clients.common.LoadTestRunner; import com.google.pubsub.clients.common.MetricsHandler.MetricName; import com.google.pubsub.clients.common.Task; @@ -36,6 +37,8 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) .batchSize(9500000) + .lingerMs(request.getPublishBatchDuration().getSeconds()) + .bufferMemory((int)Durations.toMillis(request.getPublishBatchDuration())) .isAcks(true) .build(); } @@ -60,7 +63,7 @@ public ListenableFuture doRun() { log.error(exception.getMessage(), exception); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + result.set(RunResult.fromBatchSize(batchSize)); } }); } From 89a32cafc9ed159d2257d5b67b4e2dd2e5dc2028 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 15:45:06 -0700 Subject: [PATCH 059/140] Fixed flush so it properly clears the records of the publishers. --- .../java/com/google/pubsub/clients/producer/PubsubProducer.java | 1 + .../java/com/google/pubsub/clients/tests/ProducerThread.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 8b977abb..f63eeb42 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -251,6 +251,7 @@ public void flush() { log.error("Exception occurred during flush: " + e); } } + publishers.clear(); } /** diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index 94fd05bd..f2d6606a 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -66,7 +66,6 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { log.error("Exception sending the message: " + exception.getMessage()); } else { log.info("Successfully sent message"); - log.info("HERE'S THE METADATA: " + metadata); } } } From d18794bc5d6ced91f123e79f163b27bdadb6533e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 16 Mar 2017 15:10:27 -0700 Subject: [PATCH 060/140] Modified configs to make load tests equivalent. --- .../pubsub/clients/mapped/CPSPublisherTask.java | 8 ++++---- .../pubsub/clients/producer/PubsubProducer.java | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index 13a098fe..ab31651e 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -36,9 +36,9 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) - .batchSize(9500000) - .lingerMs(request.getPublishBatchDuration().getSeconds()) - .bufferMemory((int)Durations.toMillis(request.getPublishBatchDuration())) + .batchSize(9500000L) + .lingerMs(Durations.toMillis(request.getPublishBatchDuration())) + .bufferMemory(1000000000) .isAcks(true) .build(); } @@ -63,7 +63,7 @@ public ListenableFuture doRun() { log.error(exception.getMessage(), exception); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(batchSize)); + result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); } }); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f63eeb42..100c6c17 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -28,6 +28,7 @@ import com.google.pubsub.common.PubsubChannelUtil; import com.google.pubsub.v1.TopicName; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -60,7 +61,7 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 1000; + private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; private final String project; private final Serializer keySerializer; @@ -72,7 +73,7 @@ public class PubsubProducer implements Producer { private final long lingerMs; private final Map publishers; - private boolean closed = false; + private AtomicBoolean closed; private PubsubProducer(Builder builder) { project = builder.project; @@ -84,6 +85,7 @@ private PubsubProducer(Builder builder) { lingerMs = builder.lingerMs; bufferMemory = builder.bufferMemory; publishers = new HashMap<>(); + closed = new AtomicBoolean(false); } public PubsubProducer(Map configs) { @@ -141,6 +143,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); + closed.set(false); log.debug("Producer successfully initialized."); } @@ -157,7 +160,7 @@ public Future send(ProducerRecord record) { */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); - if (closed) { + if (closed.get()) { throw new RuntimeException("Publisher is closed"); } @@ -183,7 +186,6 @@ public Future send(ProducerRecord record, Callback callbac try { Publisher newPub = Publisher.newBuilder(topic) .setBundlingSettings(BundlingSettings.newBuilder() - .setIsEnabled(true) .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) .setDelayThreshold(Duration.millis(lingerMs)) .setRequestByteThreshold(batchSize) @@ -282,6 +284,9 @@ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { throw new IllegalArgumentException("Timeout cannot be negative."); } + if (closed.getAndSet(true)) { + throw new IllegalStateException("Cannot close a producer already closed."); + } for (TopicName topic : publishers.keySet()) { try { publishers.get(topic).shutdown(); @@ -290,7 +295,6 @@ public void close(long timeout, TimeUnit unit) { } } log.debug("Closed producer"); - closed = true; } /** From 8a0c8a1679143599b9c34f19631c14c733fda2b5 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 16 Mar 2017 15:14:22 -0700 Subject: [PATCH 061/140] Fixed closed to use AtomicBoolean. --- .../pubsub/clients/producer/PubsubProducer.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f63eeb42..c439f8b3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -28,6 +28,7 @@ import com.google.pubsub.common.PubsubChannelUtil; import com.google.pubsub.v1.TopicName; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -60,7 +61,7 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 1000; + private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; private final String project; private final Serializer keySerializer; @@ -72,7 +73,7 @@ public class PubsubProducer implements Producer { private final long lingerMs; private final Map publishers; - private boolean closed = false; + private AtomicBoolean closed; private PubsubProducer(Builder builder) { project = builder.project; @@ -84,6 +85,7 @@ private PubsubProducer(Builder builder) { lingerMs = builder.lingerMs; bufferMemory = builder.bufferMemory; publishers = new HashMap<>(); + closed = new AtomicBoolean(false); } public PubsubProducer(Map configs) { @@ -141,6 +143,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); + closed = new AtomicBoolean(false); log.debug("Producer successfully initialized."); } @@ -157,7 +160,7 @@ public Future send(ProducerRecord record) { */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); - if (closed) { + if (closed.get()) { throw new RuntimeException("Publisher is closed"); } @@ -183,7 +186,6 @@ public Future send(ProducerRecord record, Callback callbac try { Publisher newPub = Publisher.newBuilder(topic) .setBundlingSettings(BundlingSettings.newBuilder() - .setIsEnabled(true) .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) .setDelayThreshold(Duration.millis(lingerMs)) .setRequestByteThreshold(batchSize) @@ -282,6 +284,9 @@ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { throw new IllegalArgumentException("Timeout cannot be negative."); } + if (closed.getAndSet(true)) { + throw new IllegalStateException("Cannot close a producer if already closed."); + } for (TopicName topic : publishers.keySet()) { try { publishers.get(topic).shutdown(); @@ -290,7 +295,6 @@ public void close(long timeout, TimeUnit unit) { } } log.debug("Closed producer"); - closed = true; } /** From bfa92b6fd2c66230cb407acc4830b036690c325f Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 14:40:32 -0700 Subject: [PATCH 062/140] Each PubsubProducer corresponds to a single topic. --- .../clients/producer/PubsubProducer.java | 91 ++++++++++--------- .../producer/PubsubProducerConfig.java | 4 + .../pubsub/clients/tests/ProducerThread.java | 2 +- .../producer/PubsubProducerConfigTest.java | 1 + 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c439f8b3..9beb478a 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -64,6 +64,7 @@ public class PubsubProducer implements Producer { private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; private final String project; + private final String topic; private final Serializer keySerializer; private final Serializer valueSerializer; private final long batchSize; @@ -71,12 +72,13 @@ public class PubsubProducer implements Producer { private final boolean isAcks; private final int maxRequestSize; private final long lingerMs; - private final Map publishers; + private Publisher publisher; private AtomicBoolean closed; private PubsubProducer(Builder builder) { project = builder.project; + topic = builder.topic; batchSize = builder.batchSize; isAcks = builder.isAcks; maxRequestSize = builder.maxRequestSize; @@ -84,7 +86,7 @@ private PubsubProducer(Builder builder) { valueSerializer = builder.valueSerializer; lingerMs = builder.lingerMs; bufferMemory = builder.bufferMemory; - publishers = new HashMap<>(); + publisher = newPublisher(); closed = new AtomicBoolean(false); } @@ -139,15 +141,36 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getLong(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + topic = configs.getString(PubsubProducerConfig.TOPIC_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); - publishers = new HashMap<>(); + publisher = newPublisher(); closed = new AtomicBoolean(false); log.debug("Producer successfully initialized."); } + private Publisher newPublisher() { + TopicName topicName = TopicName.create(project, topic); + Publisher newPub = null; + try { + newPub = Publisher.newBuilder(topicName) + .setBundlingSettings(BundlingSettings.newBuilder() + .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) + .setDelayThreshold(Duration.millis(lingerMs)) + .setRequestByteThreshold(batchSize) + .build()) + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(bufferMemory) + .build()) + .build(); + } catch (IOException e) { + log.error("Exception occurred: " + e); + } + return newPub; + } + /** * Sends the given record. */ @@ -157,6 +180,7 @@ public Future send(ProducerRecord record) { /** * Sends the given record and invokes the specified callback. + * The given record must have the same topic as the producer. */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); @@ -164,52 +188,34 @@ public Future send(ProducerRecord record, Callback callbac throw new RuntimeException("Publisher is closed"); } - TopicName topic = TopicName.create(project, record.topic()); + if (!record.topic().equals(topic)) { + throw new IllegalArgumentException("The record's topic must be the same as the Producer's topic."); + } + Map attributes = new HashMap<>(); byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic.getTopic(), record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } byte[] valueBytes = ByteString.EMPTY.toByteArray(); if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic.getTopic(), record.value()); + valueBytes = valueSerializer.serialize(topic, record.value()); } checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); - synchronized (publishers) { - if (!publishers.containsKey(topic)) { - try { - Publisher newPub = Publisher.newBuilder(topic) - .setBundlingSettings(BundlingSettings.newBuilder() - .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) - .setDelayThreshold(Duration.millis(lingerMs)) - .setRequestByteThreshold(batchSize) - .build()) - .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(bufferMemory) - .build()) - .build(); - publishers.put(topic, newPub); - } catch (IOException e) { - log.error("Exception occurred: " + e); - } - } - } - PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - - RpcFuture messageIdFuture = publishers.get(topic).publish(message); + RpcFuture messageIdFuture = publisher.publish(message); Future future = SettableFuture.create(); - RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic.getTopic(), 0), + RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic, 0), 0, 0, System.currentTimeMillis(), 0, serializedKey.length, valueBytes.length); if (callback != null) { @@ -246,14 +252,13 @@ private void checkRecordSize(int size) { */ public void flush() { log.debug("Flushing..."); - for (TopicName topic : publishers.keySet()) { - try { - publishers.get(topic).shutdown(); - } catch (Exception e) { - log.error("Exception occurred during flush: " + e); - } + // Shut down publisher to flush the logs, then immediately restart it. + try { + publisher.shutdown(); + publisher = newPublisher(); + } catch (Exception e) { + log.error("Exception occurred during flush: " + e); } - publishers.clear(); } /** @@ -287,12 +292,10 @@ public void close(long timeout, TimeUnit unit) { if (closed.getAndSet(true)) { throw new IllegalStateException("Cannot close a producer if already closed."); } - for (TopicName topic : publishers.keySet()) { - try { - publishers.get(topic).shutdown(); - } catch (Exception e) { - log.error("Exception occurred during close: " + e); - } + try { + publisher.shutdown(); + } catch (Exception e) { + log.error("Exception occurred during close: " + e); } log.debug("Closed producer"); } @@ -306,6 +309,7 @@ public static class Builder { private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; + private final String topic; private long batchSize; private long lingerMs; @@ -313,12 +317,13 @@ public static class Builder { private boolean isAcks; private int maxRequestSize; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, String topic, Serializer keySerializer, Serializer valueSerializer) { Preconditions .checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; + this.topic = topic; setDefaults(); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 505fda20..772ca0fb 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -51,6 +51,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String TOPIC_CONFIG = "topic"; + private static final String TOPIC_DOC = "Topic to which messages are published."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; @@ -76,6 +79,7 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, DEFAULT_BUFFER_MEMORY, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(TOPIC_CONFIG, Type.STRING, Importance.HIGH, TOPIC_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) .define(LINGER_MS_CONFIG, Type.LONG, DEFAULT_LINGER_MS, atLeast(0L), Importance.MEDIUM, LINGER_MS_DOC) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index f2d6606a..95d83999 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -39,7 +39,7 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic, int numMessages) { this.command = s; - this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + this.producer = new Builder<>(props.getProperty("project"), topic, new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .lingerMs(Long.parseLong(props.getProperty("linger.ms"))) diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java index 000536cd..4baefb35 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -24,6 +24,7 @@ public void testSuccessAllConfigsProvided() { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("project", "unit-test-project") + .put("topic", "unit-test-topic") .build() ); From 6ab4942997835a63a50f46ae5a70a793e136c5e6 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 16:37:00 -0700 Subject: [PATCH 063/140] Modified mapped/CPSPublisherTask to reflect updates in PubsubProducer. --- .../google/pubsub/clients/mapped/CPSPublisherTask.java | 2 +- .../pubsub/clients/producer/PubsubProducerTest.java | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index ab31651e..290d0c48 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -34,7 +34,7 @@ private CPSPublisherTask(StartRequest request) { this.payload = LoadTestRunner.createMessage(request.getMessageSize()); this.batchSize = request.getPublishBatchSize(); - this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), + this.publisher = new PubsubProducer.Builder<>(request.getProject(), topic, new StringSerializer(), new StringSerializer()) .batchSize(9500000L) .lingerMs(Durations.toMillis(request.getPublishBatchDuration())) diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index de8c1ae6..a56e9d41 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -113,12 +113,4 @@ public void testCloseChannelCloseSuccessful() { } - private PubsubProducer getNewProducer() throws IOException { - - return new Builder<>(PROJECT, testSerializer, testSerializer) - .publisherFutureStub(mockStub) - .pubsubChannelUtil(mockChannelUtil) - .build(); - } - } From e51ac98ffc3e578e50d965f32ceebfa80010f266 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 16:40:08 -0700 Subject: [PATCH 064/140] Deleted unit test classes that are empty/no longer in use. --- .../clients/producer/PubsubProducerTest.java | 116 ------------------ .../pubsub/common/PubsubChannelUtilTest.java | 51 -------- 2 files changed, 167 deletions(-) delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index a56e9d41..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.powermock.api.easymock.PowerMock.createMock; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; -import java.io.IOException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -//@RunWith(PowerMockRunner.class) -//@PrepareForTest(PublisherFutureStub.class) -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - private static final String PROJECT = "unit-test-proj"; - private static final StringSerializer testSerializer = new StringSerializer(); - private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); - - private static PublisherFutureStub mockStub; - @Mock private static PubsubChannelUtil mockChannelUtil; - - private boolean channelIsClosed; - - - @Before - public void setUp() { - // mockStub = createMock(PublisherFutureStub.class); - MockitoAnnotations.initMocks(this); - channelIsClosed = false; - - Mockito.doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - return channelIsClosed = true; - } - }).when(mockChannelUtil).closeChannel(); - } - - /* send() tests */ - @Test - public void testSendPublisherClosed() throws IOException { - /* PubsubProducer testProducer = getNewProducer(); - testProducer.close(); - - try { - testProducer.send(testRecord); - fail("Should have thrown a runtime exception."); - } catch (RuntimeException expected) { - assertTrue("Channel should be closed.", channelIsClosed); - }*/ - } - - @Test - public void testSendRecordTooLarge() { - - } - - @Test - public void testSendBatchFull() { - - } - - /* This happens when the batch size is > 1 and not enough messages are batched */ - @Test - public void testSendMessageNotSentYet() { - - } - - /* flush() tests */ - @Test - public void testFlushMessagesSent() { - - } - - /* close() tests */ - @Test - public void testCloseTimeoutLessThanZero() { - - } - - @Test - public void testCloseChannelCloseSuccessful() { - - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java deleted file mode 100644 index 2c5ad730..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.pubsub.common; - -import static org.junit.Assert.assertNotNull; - -import java.io.IOException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PubsubChannelUtilTest { - - private static PubsubChannelUtil channelUtil; - - @BeforeClass - public static void setUp() throws IOException { - channelUtil = new PubsubChannelUtil(); - } - - @AfterClass - public static void tearDown() { - channelUtil.closeChannel(); - } - - @Test - public void testGetCallCredentials() throws IOException { - assertNotNull(channelUtil.callCredentials()); - channelUtil.closeChannel(); - } - - @Test - public void testGetChannel() { - assertNotNull(channelUtil.channel()); - } -} From ab9274909e1fed92aee123a7c8e3f51240f17d55 Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 065/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From 65ee7c3f78fcbd026cf9e444f2e51912eb9eaa36 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 066/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From 9d48d45967788aa21a647cd9a355d9789f95f6a9 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 067/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From fd330ebcd9386f8619ca427462c2f0befe6ce020 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 068/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From a13d0cb390a501f458d8aa2225d85a69112616cd Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 069/140] Added PubsubConsumer class --- load-test-framework/flic.iml | 117 +--- .../clients/consumer/PubsubConsumer.java | 651 ++++++++++++++++++ 2 files changed, 654 insertions(+), 114 deletions(-) diff --git a/load-test-framework/flic.iml b/load-test-framework/flic.iml index 6826b98d..19dbd15d 100644 --- a/load-test-framework/flic.iml +++ b/load-test-framework/flic.iml @@ -1,117 +1,6 @@ - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 42b3f7db..8ffc2684 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -109,4 +109,655 @@ public PubsubConsumer(Properties properties, private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { } +======= +package com.google.kafka.cients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file From f5fae545a40fe0eadc4b531808c5e04bd7977121 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 070/140] New maven setup for mapped api --- pubsub-mapped-api/pom.xml | 49 ++++ .../clients/consumer/PubsubConsumer.java | 230 ++++++++++-------- .../clients/producer/PubsubProducer.java | 2 +- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 0 .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 10 files changed, 178 insertions(+), 103 deletions(-) create mode 100644 pubsub-mapped-api/pom.xml rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/consumer/PubsubConsumer.java (83%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/PubsubProducer.java (99%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/PubsubProducerTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml new file mode 100644 index 00000000..1bb14363 --- /dev/null +++ b/pubsub-mapped-api/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + com.google.pubsub + pubsub-mapped-api + jar + 1.0-SNAPSHOT + MappedApi + http://maven.apache.org + + + junit + junit + 4.12 + + + org.apache.kafka + kafka_2.10 + 0.10.0.0 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.slf4j + slf4j-api + 1.7.21 + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java similarity index 83% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 8ffc2684..60084a19 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -12,105 +12,58 @@ */ package com.google.kafka.clients.consumer; -public class PubsubConsumer implements Consumer { - - private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; // this might need to be an /internal class - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - // currentThread holds the threadId of the current thread accessing PubsubConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } -======= -package com.google.kafka.cients.consumer; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.network.ChannelBuilder; +import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -205,7 +158,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { + /* try { log.debug("Starting the Kafka consumer"); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); @@ -244,9 +197,82 @@ private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, D this.keyDeserializer = keyDeserializer; } if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; } - } // left off at kafka's line 645 + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ } /** diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java similarity index 99% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 2077a3c1..5f7d3507 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -85,7 +85,7 @@ public class PubsubProducer implements Producer { private final ProducerConfig producerConfig; private final long maxBlockTimeMs; private final int requestTimeoutMs; - private final ProducerInterceptors interceptors + private final ProducerInterceptors interceptors; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 35087f8f3f963afcd12a1f78f5571c6d5d5b863e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 071/140] scratching some of the mapped api to make it more pub/sub --- pubsub-mapped-api/pom.xml | 20 + .../clients/consumer/PubsubConsumer.java | 120 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 6 files changed, 25 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1bb14363..ef3ab09f 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -33,6 +33,26 @@ slf4j-log4j12 1.7.21 + + io.grpc + grpc-all + 1.0.1 + + + io.grpc + grpc-netty + 1.0.1 + + + io.grpc + grpc-protobuf + 1.0.1 + + + io.grpc + grpc-stub + 1.0.1 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 60084a19..1149cc4c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -10,7 +10,8 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; + +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -158,121 +159,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -325,7 +212,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From 63b340c51b7d5cea762c6ee1aae64a71d2cd4c75 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 072/140] Implementing publisher/producer using grpc backend. --- pubsub-mapped-api/pom.xml | 64 +- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../producer/PubsubProducerConfig.java | 85 ++ .../com/google/pubsub/common/PubsubUtils.java | 46 + .../src/main/resources/log4j.properties | 7 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 11 files changed, 1061 insertions(+), 1852 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java create mode 100644 pubsub-mapped-api/src/main/resources/log4j.properties delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index ef3ab09f..916913ca 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,6 +8,11 @@ MappedApi http://maven.apache.org + + com.google.pubsub + cloud-pubsub-client + 0.2-EXPERIMENTAL + junit junit @@ -16,7 +21,7 @@ org.apache.kafka kafka_2.10 - 0.10.0.0 + 0.10.1.1 org.apache.commons @@ -33,6 +38,11 @@ slf4j-log4j12 1.7.21 + + log4j + log4j + 1.2.17 + io.grpc grpc-all @@ -55,10 +65,62 @@ + + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + package + + shade + + + false + + + + com.google.pubsub.clients.ClientMain + + + mapped-api + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + + com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:0.14.0:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto/ + + + + + compile + compile-custom + + + + org.apache.maven.plugins maven-compiler-plugin + 3.6.1 1.8 1.8 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 1149cc4c..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,30 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -33,7 +34,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -68,608 +68,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java new file mode 100644 index 00000000..aa812494 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients.producer; + +import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.serialization.Serializer; + +public class PubsubProducerConfig extends AbstractConfig { + private static final ConfigDef CONFIG; + + public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + + public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + + public static final String BATCH_SIZE_CONFIG = "batch.size"; + private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; + + public static final String ACKS_CONFIG = "acks"; + private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project to use with the publisher."; + + static { + CONFIG = + new ConfigDef() + .define( + KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) + .define( + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + } + + PubsubProducerConfig(Map properties) { + super(CONFIG, properties); + } + + public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + newConfigs.putAll(configs); + if (keySerializer != null) { + newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newConfigs.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newConfigs; + } + + public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + Map newConfigs = new HashMap(); + Properties newProperties = new Properties(); + if (keySerializer != null) { + newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); + } + if (valueSerializer != null) { + newProperties.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); + } + return newProperties; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/resources/log4j.properties b/pubsub-mapped-api/src/main/resources/log4j.properties new file mode 100644 index 00000000..ff0cb5bd --- /dev/null +++ b/pubsub-mapped-api/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=DEBUG, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + +log4j.org.apache.kafka = OFF +log4j.logger.io.grpc = OFF +log4j.logger.io.netty = OFF \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From d40d37cde20924d4478e8eb9856c2f3f60a5a415 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 073/140] Add details to simplified producer --- pubsub-mapped-api/pom.xml | 5 ++ .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../producer/PubsubProducerConfig.java | 6 +- .../internals/PubsubFutureRecordMetadata.java | 38 +++++++++++++ 4 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 916913ca..38b17362 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.cloud + google-cloud-pubsub + 0.9.2-alpha + junit junit diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index aa812494..5cee6425 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -43,6 +43,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; + private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + static { CONFIG = new ConfigDef() @@ -52,7 +55,8 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java new file mode 100644 index 00000000..88a5fd90 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +/*package com.google.pubsub.clients.producer.internals; + +private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; + } + + public boolean isCancelled() { + return false; + } + + public boolean isDone() { + return false; + } + + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; + } + + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +}*/ \ No newline at end of file From 7cf310800eb00a838b57ed85ef6267e9104b314b Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 074/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 12 +++ .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 56 +++++++++++ .../pubsub/clients/ProducerThreadPool.java | 72 ++++++++++++++ .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 12 ++- ...ubsubUtils.java => PubsubChannelUtil.java} | 35 +++++-- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 8 files changed, 228 insertions(+), 178 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename pubsub-mapped-api/src/main/java/com/google/pubsub/common/{PubsubUtils.java => PubsubChannelUtil.java} (66%) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 38b17362..99f6ff22 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,6 +23,18 @@ junit 4.12 + + org.powermock + powermock-module-junit4-legacy + 1.7.0RC2 + test + + + org.powermock + powermock-api-easymock + 1.7.0RC2 + test + org.apache.kafka kafka_2.10 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java new file mode 100644 index 00000000..092ac653 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -0,0 +1,56 @@ +package com.google.pubsub.clients; + +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; + + +public class ProducerThread implements Runnable { + private String command; + private PubsubProducer producer; + private String topic; + private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + + public ProducerThread(String s, Properties props, String topic) { + this.command = s; + this.producer = new PubsubProducer<>(props); + this.topic = topic; + } + + public void run() { + log.info("Start running the command"); + processCommand(); + log.info("End running the command"); + } + + private void processCommand() { + try { + ProducerRecord msg = new ProducerRecord(topic, "message" + command); + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } + } + ); + Thread.sleep(5000); + producer.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public String toString() { + return command; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java new file mode 100644 index 00000000..b37cba9e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -0,0 +1,72 @@ +package com.google.pubsub.clients; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.pubsub.clients.producer.PubsubProducer; +import java.util.Properties; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class ProducerThreadPool { + private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); + + public static void main(String[] args) { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); + threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread t, Throwable e) { + log.error(t + " throws exception: " + e); + } + }); + + //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); + ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); + + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + + /* props.putAll(new ImmutableMap.Builder<>() + .put("max.block.ms", "30000") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("bootstrap.servers", "104.198.72.101:9092") + .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB + // 10M, high enough to allow for duration to control batching + .put("batch.size", Integer.toString(10 * 1000 * 1000)) + .put("linger.ms", 10) + .build() + );*/ + + for (int i = 0; i < 20; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0]); + executor.execute(worker); + } + + executor.shutdown(); + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + log.error("Executor did not terminate"); + } + } + } catch (InterruptedException ie) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..bcf86059 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null()); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 88a5fd90..771fcb82 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -13,9 +13,15 @@ * the License. */ -/*package com.google.pubsub.clients.producer.internals; +package com.google.pubsub.clients.producer.internals; -private static class PubsubFutureRecordMetadata implements Future { +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.producer.RecordMetadata; + +public class PubsubFutureRecordMetadata implements Future { public boolean cancel(boolean b) { return false; } @@ -35,4 +41,4 @@ public RecordMetadata get() throws InterruptedException, ExecutionException { public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } -}*/ \ No newline at end of file +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 10a5599e..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,31 +16,46 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; import io.grpc.auth.ClientAuthInterceptor; import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.concurrent.Executors; -public class PubsubUtils { +public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); + + private ManagedChannel channel; + private CallCredentials callCredentials; + + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + callCredentials = MoreCallCredentials.from(credentials); + channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + } + + public CallCredentials callCredentials() { + return callCredentials; + } + + public Channel channel() { + return channel; + } + + public void closeChannel() { + channel.shutdown(); } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From 6410c90a3b5ea607bec97344e29391ec1f4f27ca Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 075/140] Working on unit tests, need to mock the publisher calls --- pubsub-mapped-api/pom.xml | 17 ++--- .../google/pubsub/clients/ProducerThread.java | 26 ++++---- .../pubsub/clients/ProducerThreadPool.java | 10 +-- .../clients/producer/PubsubProducer.java | 30 ++------- .../producer/PubsubProducerConfig.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- .../pubsub/common/PubsubChannelUtilTest.java | 51 +++++++++++++++ 8 files changed, 134 insertions(+), 77 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 99f6ff22..1fdd87b7 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,18 +23,6 @@ junit 4.12 - - org.powermock - powermock-module-junit4-legacy - 1.7.0RC2 - test - - - org.powermock - powermock-api-easymock - 1.7.0RC2 - test - org.apache.kafka kafka_2.10 @@ -80,6 +68,11 @@ grpc-stub 1.0.1 + + org.mockito + mockito-all + 2.0.2-beta + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 092ac653..e075ae36 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -30,19 +30,21 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord(topic, "message" + command); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } } - } - } - ); + ); + } Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index b37cba9e..4b168cc0 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -33,15 +33,15 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 1) + .put("batch.size", 20) .put("linger.ms", 1) .build() ); /* props.putAll(new ImmutableMap.Builder<>() .put("max.block.ms", "30000") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") .put("bootstrap.servers", "104.198.72.101:9092") .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB @@ -49,9 +49,9 @@ public void uncaughtException(Thread t, Throwable e) { .put("batch.size", Integer.toString(10 * 1000 * 1000)) .put("linger.ms", 10) .build() - );*/ + ); */ - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 1; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index bcf86059..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -273,7 +251,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null()); + doSend(request, null, null); } } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 5cee6425..ff3e6139 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -76,8 +76,8 @@ public static Map addSerializerToConfig(Map conf } public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap(); Properties newProperties = new Properties(); + newProperties.putAll(properties); if (keySerializer != null) { newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java new file mode 100644 index 00000000..2c5ad730 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.common; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PubsubChannelUtilTest { + + private static PubsubChannelUtil channelUtil; + + @BeforeClass + public static void setUp() throws IOException { + channelUtil = new PubsubChannelUtil(); + } + + @AfterClass + public static void tearDown() { + channelUtil.closeChannel(); + } + + @Test + public void testGetCallCredentials() throws IOException { + assertNotNull(channelUtil.callCredentials()); + channelUtil.closeChannel(); + } + + @Test + public void testGetChannel() { + assertNotNull(channelUtil.channel()); + } +} From 8eff41b51299db808b7e856847396cdf7ea2a51a Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 076/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 6 +- .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../producer/PubsubProducerConfig.java | 10 +- .../pubsub/common/PubsubChannelUtil.java | 4 - .../producer/PubsubProducerConfigTest.java | 59 +++++++++++ .../clients/producer/PubsubProducerTest.java | 35 +----- 6 files changed, 166 insertions(+), 48 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index e075ae36..2a3bb585 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -2,6 +2,7 @@ import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; +import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; @@ -18,7 +19,10 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic) { this.command = s; - this.producer = new PubsubProducer<>(props); + //this.producer = new PubsubProducer<>(props); + this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) + .batchSize(props.getProperty("batch.size")) + .) this.topic = topic; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index ff3e6139..f03c017f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -46,6 +46,10 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final boolean DEFAULT_ACKS = true; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + static { CONFIG = new ConfigDef() @@ -53,10 +57,10 @@ public class PubsubProducerConfig extends AbstractConfig { KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java new file mode 100644 index 00000000..000536cd --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -0,0 +1,59 @@ +package com.google.pubsub.clients.producer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Properties; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.serialization.Serializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + + +public class PubsubProducerConfigTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testSuccessAllConfigsProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "unit-test-project") + .build() + ); + + PubsubProducerConfig testConfig = new PubsubProducerConfig(props); + + assertEquals("Project config equals unit-test-project.", "unit-test-project", testConfig.getString(PubsubProducerConfig.PROJECT_CONFIG)); + assertNotNull("Key serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + assertNotNull("Value serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + } + + @Test + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "unit-test-project"); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + + } + + @Test + public void testNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + exception.expect(ConfigException.class); + new PubsubProducerConfig(props); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From 252ac5f9f1f6442555dff5c68f4cd2df4eaf5a6b Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 077/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 16 +++++++++------- .../pubsub/clients/ProducerThreadPool.java | 7 ++++--- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 2a3bb585..d1ae43fa 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,6 +1,8 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; @@ -17,12 +19,12 @@ public class ProducerThread implements Runnable { private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); - public ProducerThread(String s, Properties props, String topic) { + public ProducerThread(String s, Properties props, String topic) throws IOException { this.command = s; - //this.producer = new PubsubProducer<>(props); - this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) - .batchSize(props.getProperty("batch.size")) - .) + this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + .batchSize(Integer.parseInt(props.getProperty("batch.size"))) + .isAcks(props.getProperty("acks").matches("1|all")) + .build(); this.topic = topic; } @@ -34,8 +36,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 4b168cc0..edb9707b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -1,5 +1,6 @@ package com.google.pubsub.clients; +import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -15,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -33,8 +34,8 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 20) - .put("linger.ms", 1) + .put("batch.size", "1") + .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From f8bada3a95f58d80e2e98f205b71c43f5ed68a6c Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 08:32:09 -0800 Subject: [PATCH 078/140] Mapped publisher task for load testing. --- .../clients/mapped/MappedPublisherTask.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java new file mode 100644 index 00000000..868fe2d3 --- /dev/null +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -0,0 +1,71 @@ +package com.google.pubsub.clients.mapped; + +import com.beust.jcommander.JCommander; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.pubsub.clients.common.LoadTestRunner; +import com.google.pubsub.clients.common.MetricsHandler; +import com.google.pubsub.clients.common.MetricsHandler.MetricName; +import com.google.pubsub.clients.common.Task; +import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.flic.common.LoadtestProto.StartRequest; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer + * interface + */ +public class MappedPublisherTask extends Task { + + private static final Logger log = LoggerFactory.getLogger(MappedPublisherTask.class); + private final String topic; + private final String payload; + private final int batchSize; + private final PubsubProducer publisher; + + @SuppressWarnings("unchecked") + private MappedPublisherTask(StartRequest request) throws IOException { + super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); + this.topic = request.getTopic(); + this.payload = LoadTestRunner.createMessage(request.getMessageSize()); + this.batchSize = request.getPublishBatchSize(); + + this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), + new StringSerializer()) + .batchSize(this.batchSize) + .isAcks(true) + .build(); + } + + public static void main(String[] args) throws Exception { + LoadTestRunner.Options options = new LoadTestRunner.Options(); + new JCommander(options, args); + LoadTestRunner.run(options, MappedPublisherTask::new); + } + + @Override + public ListenableFuture doRun() { + SettableFuture result = SettableFuture.create(); + AtomicInteger messagesToSend = new AtomicInteger(batchSize); + AtomicInteger messagesSentSuccess = new AtomicInteger(batchSize); + for (int i = 0; i < batchSize; i++) { + publisher.send( + new ProducerRecord<>(topic, null, System.currentTimeMillis(), null, payload), + (metadata, exception) -> { + if (exception != null) { + messagesSentSuccess.decrementAndGet(); + log.error(exception.getMessage(), exception); + } + if (messagesToSend.decrementAndGet() == 0) { + result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + } + }); + } + return result; + } +} From ba63438d2534e4ea75128ca2fda62e3cc39b6452 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 13:32:36 -0800 Subject: [PATCH 079/140] Fixing dependencies between load-test-framework and mapped-api --- load-test-framework/pom.xml | 5 + .../clients/mapped/MappedPublisherTask.java | 3 +- .../pubsub/flic/controllers/Client.java | 5 +- pubsub-mapped-api/pom.xml | 17 ++- .../clients/producer/PubsubProducer.java | 3 +- .../pubsub/common/PubsubChannelUtil.java | 17 ++- .../clients/producer/PubsubProducerTest.java | 114 ++++++++++++++---- 7 files changed, 128 insertions(+), 36 deletions(-) diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 582821bd..1a443a4e 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -13,6 +13,11 @@ cloud-pubsub-client 0.2-EXPERIMENTAL + + com.google.pubsub + pubsub-mapped-api + 1.0-SNAPSHOT + junit junit diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java index 868fe2d3..a9410668 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -4,7 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler; import com.google.pubsub.clients.common.MetricsHandler.MetricName; import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.producer.PubsubProducer; @@ -29,7 +28,7 @@ public class MappedPublisherTask extends Task { private final PubsubProducer publisher; @SuppressWarnings("unchecked") - private MappedPublisherTask(StartRequest request) throws IOException { + private MappedPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); this.payload = LoadTestRunner.createMessage(request.getMessageSize()); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index a246675d..78285efe 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -116,6 +116,8 @@ public static String getTopicSuffix(ClientType clientType) { case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; + case MAPPED_PUBLISHER: + return "mapped"; } return null; } @@ -301,7 +303,8 @@ public enum ClientType { CPS_GCLOUD_PYTHON_PUBLISHER, CPS_VTK_JAVA_PUBLISHER, KAFKA_PUBLISHER, - KAFKA_SUBSCRIBER; + KAFKA_SUBSCRIBER, + MAPPED_PUBLISHER; public boolean isCpsPublisher() { switch (this) { diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 1fdd87b7..d9e7639a 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -28,6 +28,21 @@ kafka_2.10 0.10.1.1 + + org.powermock + powermock-module-junit4 + 1.6.6 + + + org.powermock + powermock-api-mockito + 1.6.6 + + + org.powermock + powermock-api-easymock + 1.6.6 + org.apache.commons commons-lang3 @@ -71,7 +86,7 @@ org.mockito mockito-all - 2.0.2-beta + 1.9.5 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..48eba14c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -27,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -360,7 +359,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..a6d14269 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,8 +16,6 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; @@ -27,10 +25,12 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { - + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +41,15 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault() + .createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 32555be2..de8c1ae6 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,45 +16,109 @@ */ package com.google.pubsub.clients.producer; -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.powermock.api.easymock.PowerMock.createMock; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import java.io.IOException; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - +//@RunWith(PowerMockRunner.class) +//@PrepareForTest(PublisherFutureStub.class) public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; + private static final String PROJECT = "unit-test-proj"; + private static final StringSerializer testSerializer = new StringSerializer(); + private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); + + private static PublisherFutureStub mockStub; + @Mock private static PubsubChannelUtil mockChannelUtil; + + private boolean channelIsClosed; + + + @Before + public void setUp() { + // mockStub = createMock(PublisherFutureStub.class); + MockitoAnnotations.initMocks(this); + channelIsClosed = false; + + Mockito.doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + return channelIsClosed = true; + } + }).when(mockChannelUtil).closeChannel(); + } /* send() tests */ @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub + public void testSendPublisherClosed() throws IOException { + /* PubsubProducer testProducer = getNewProducer(); + testProducer.close(); + + try { + testProducer.send(testRecord); + fail("Should have thrown a runtime exception."); + } catch (RuntimeException expected) { + assertTrue("Channel should be closed.", channelIsClosed); + }*/ + } + + @Test + public void testSendRecordTooLarge() { + + } + + @Test + public void testSendBatchFull() { + + } + + /* This happens when the batch size is > 1 and not enough messages are batched */ + @Test + public void testSendMessageNotSentYet() { + + } + + /* flush() tests */ + @Test + public void testFlushMessagesSent() { + + } + + /* close() tests */ + @Test + public void testCloseTimeoutLessThanZero() { + + } + + @Test + public void testCloseChannelCloseSuccessful() { - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal } - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); + private PubsubProducer getNewProducer() throws IOException { - return new PubsubProducer(props); + return new Builder<>(PROJECT, testSerializer, testSerializer) + .publisherFutureStub(mockStub) + .pubsubChannelUtil(mockChannelUtil) + .build(); } } From 2ecd0bbc9af64c9c27d3d597957d58e58ee29776 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 3 Mar 2017 16:49:51 -0800 Subject: [PATCH 080/140] Fixed unit tests to be compatible with my load test changes. --- .../pubsub/clients/mapped/MappedPublisherTask.java | 1 - .../main/java/com/google/pubsub/flic/Driver.java | 13 ++++++++++++- .../com/google/pubsub/flic/controllers/Client.java | 3 +++ .../google/pubsub/flic/output/SheetsService.java | 2 ++ .../pubsub/flic/output/SheetsServiceTest.java | 5 ++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java index a9410668..ffb01012 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java @@ -8,7 +8,6 @@ import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.producer.PubsubProducer; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.StringSerializer; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 04dc5af6..3fa0803f 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -130,6 +130,13 @@ public class Driver { ) private int kafkaSubscriberCount = 0; + @Parameter( + names = {"--mapped_publisher_count"}, + description = "Number of mapped publishers to start." + ) + + private int mappedPublisherCount = 0; + @Parameter( names = {"--message_size", "-m"}, description = "Message size in bytes (only when publishing messages).", @@ -372,6 +379,10 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } + if (mappedPublisherCount > 0) { + clientParamsMap.put( + new ClientParams(ClientType.MAPPED_PUBLISHER, null), mappedPublisherCount); + } if (kafkaPublisherCount > 0) { clientParamsMap.put( new ClientParams(ClientType.KAFKA_PUBLISHER, null), kafkaPublisherCount); @@ -402,7 +413,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + mappedPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index 78285efe..68f34b47 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -198,6 +198,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case MAPPED_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -334,6 +335,7 @@ public boolean isPublisher() { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: case KAFKA_PUBLISHER: + case MAPPED_PUBLISHER: return true; default: return false; @@ -347,6 +349,7 @@ public ClientType getSubscriberType() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case MAPPED_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 1e47aaec..08718d2c 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -170,6 +170,8 @@ public List>> getValuesList(Map> types = new HashMap<>(); int expectedCpsCount = 0; int expectedKafkaCount = 0; + int expectedMappedCount = 0; Map paramsMap = new HashMap<>(); for (ClientType type : ClientType.values()) { paramsMap.put(new ClientParams(type, ""), 1); @@ -44,8 +45,10 @@ public void testClientSwitch() { expectedCpsCount++; } else if (type.toString().startsWith("kafka")) { expectedKafkaCount++; + } else if (type.toString().startsWith("mapped")) { + expectedMappedCount++; } else { - fail("ClientType toString didn't start with cps or kafka"); + fail("ClientType toString didn't start with cps, mapped, or kafka"); } } types.put("zone-test", paramsMap); From 79d34db0ebb5a76f5c87517c45c0da38414197a2 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:19:31 -0800 Subject: [PATCH 081/140] Updated pom for load tests --- load-test-framework/flic.iml | 6 ------ load-test-framework/pom.xml | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 load-test-framework/flic.iml diff --git a/load-test-framework/flic.iml b/load-test-framework/flic.iml deleted file mode 100644 index 19dbd15d..00000000 --- a/load-test-framework/flic.iml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 1a443a4e..21047f54 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -155,6 +155,11 @@ zkclient 0.7 + + org.apache.kafka + kafka-clients + 0.10.0.0 + From e8713a2c14e32b985512ba4f9e93670e5b80bab9 Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 082/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From 0549394fdacac07fa455a2c6b9f9ce06df76dbb9 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 083/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From 03204b833eaa4429781f3d5634fd7e30823c7bab Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 084/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From 5c64da28e5b72bcfbe997c1091d9fc2537fdfae4 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 085/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From 9f57550df05ab0b8ddf3ea95bb3cee75a68c9e7e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 086/140] Added PubsubConsumer class --- .../clients/consumer/PubsubConsumer.java | 743 +++++++++++++++--- 1 file changed, 647 insertions(+), 96 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 42b3f7db..12bb1ec5 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -10,103 +10,654 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; +package com.google.kafka.cients.consumer; public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; // this might need to be an /internal class - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - // currentThread holds the threadId of the current thread accessing PubsubConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file From a1bb29079e2fc8075a48179bd1ac560db475302c Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 087/140] New maven setup for mapped api --- .../clients/consumer/PubsubConsumer.java | 663 --------- .../clients/producer/PubsubProducer.java | 656 --------- .../clients/producer/PubsubProducerTest.java | 113 -- .../clients/consumer/PubsubConsumer.java | 1277 ++++++++++------- .../clients/producer/PubsubProducer.java | 910 +++++++----- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 187 ++- .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 12 files changed, 1410 insertions(+), 2396 deletions(-) delete mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java delete mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java delete mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java deleted file mode 100644 index 12bb1ec5..00000000 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ /dev/null @@ -1,663 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.cients.consumer; - -public class PubsubConsumer implements Consumer { - - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured - } - } // left off at kafka's line 645 - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } -} \ No newline at end of file diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java deleted file mode 100644 index 2077a3c1..00000000 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ /dev/null @@ -1,656 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer; - -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; -import org.apache.kafka.common.utils.SystemTime; -import org.apache.kafka.common.utils.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.SocketOptions; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A Kafka client that publishes records to Google Cloud Pub/Sub. - */ -public class PubsubProducer implements Producer { - - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); - } - - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } - } - - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); - } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); - } - - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } - } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); - } - - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } - - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; - } - - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); - } - - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); - } - - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); - } - - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } - - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - } - - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } - - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } - } -} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java deleted file mode 100644 index fd8e96b4..00000000 --- a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.kafka.clients.producer; - -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") -public class PubsubProducerTest { - - @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); - - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); - - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 285c5d8e..cadfbf1b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,31 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ +<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +package com.google.kafka.cients.consumer; +======= -package com.google.pubsub.clients.consumer; +package com.google.kafka.clients.consumer; -import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.OffsetAndTimestamp; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -34,6 +36,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -68,523 +71,723 @@ public class PubsubConsumer implements Consumer { - public PubsubConsumer(Map configs) { - - } - - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - public PubsubConsumer(Properties properties) { - - } - - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, - OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public OffsetAndMetadata committed(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the metrics kept by the consumer - */ - public Map metrics() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public List partitionsFor(String topic) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public Map> listTopics() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - public void pause(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - public void resume(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - public Set paused() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - public Map offsetsForTimes(Map timestampsToSearch) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - public Map beginningOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - public Map endOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - public void wakeup() { - throw new NotImplementedException("Not yet implemented"); - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + /* try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 48eba14c..5f7d3507 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,402 +1,656 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ - -package com.google.pubsub.clients.producer; - -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubChannelUtil; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; -import org.apache.kafka.clients.producer.internals.ProduceRequestResult; -import org.apache.kafka.clients.producer.internals.RecordAccumulator; -import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; +import java.net.SocketOptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - - private final PublisherFutureStub publisher; - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final int batchSize; - private final boolean isAcks; - private final Map> perTopicBatches; - private final int maxRequestSize; - private final Time time; - private final PubsubChannelUtil channelUtil; - - private boolean closed = false; - - private PubsubProducer(Builder builder) { - publisher = builder.publisher; - project = builder.project; - batchSize = builder.batchSize; - isAcks = builder.isAcks; - perTopicBatches = builder.perTopicBatches; - maxRequestSize = builder.maxRequestSize; - time = builder.time; - channelUtil = builder.channelUtil; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; - } - - public PubsubProducer(Map configs) { - this(new PubsubProducerConfig(configs), null, null); - } - - public PubsubProducer(Map configs, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - public PubsubProducer(Properties properties) { - this(new PubsubProducerConfig(properties), null, null); - } - - public PubsubProducer(Properties properties, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { - try { - log.trace("Starting the Pubsub producer"); - this.time = new SystemTime(); - channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); - - if (keySerializer == null) { - this.keySerializer = - configs.getConfiguredInstance( - PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.keySerializer.configure(configs.originals(), true); - } else { - configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - - if (valueSerializer == null) { - this.valueSerializer = configs.getConfiguredInstance( - PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.valueSerializer.configure(configs.originals(), false); - } else { - configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - } catch (Exception e) { - throw new RuntimeException(e); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors; + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); } - batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); - isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); - project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - - log.debug("Producer successfully initialized."); - } - - /** - * Send the given record asynchronously and return a future which will eventually contain the response information. - * - * @param record The record to send - * @return A future which will eventually contain the response information - */ - public Future send(ProducerRecord record) { - return send(record, null); - } - - /** - * Send a record and invoke the given callback when the record has been acknowledged by the server - */ - public Future send(ProducerRecord record, Callback callback) { - log.trace("Received " + record.toString()); - if (closed) { - throw new RuntimeException("Publisher is closed"); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); } - String topic = record.topic(); - Map attributes = new HashMap<>(); - - byte[] serializedKey = ByteString.EMPTY.toByteArray(); - if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); } - if (project == null) { - throw new RuntimeException("No project specified."); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); } - byte[] valueBytes = ByteString.EMPTY.toByteArray(); - if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic, record.value()); - } + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } - checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); - - PubsubMessage message = - PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(valueBytes)) - .putAllAttributes(attributes) - .build(); - List batch = perTopicBatches.get(topic); - if (batch == null) { - batch = new ArrayList<>(batchSize); - perTopicBatches.put(topic, batch); - } - batch.add(message); - - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - RecordAccumulator.RecordAppendResult result = new RecordAppendResult( - new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); - if (result.batchIsFull) { - log.trace("Sending a batch of messages."); - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); - doSend(request, callback, result); - } - return result.future; - } - - private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { - try { - ListenableFuture response = publisher.publish(request); - if (callback != null) { - if (isAcks) { - Futures.addCallback( - response, - new FutureCallback() { - public void onSuccess(PublishResponse response) { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } - public void onFailure(Throwable t) { - callback.onCompletion(null, new Exception(t)); - } - } - ); + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); } else { - perTopicBatches.clear(); - callback.onCompletion(null, null); + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); } - } else { - response.get(); - perTopicBatches.clear(); - } - } catch (InterruptedException | ExecutionException e) { - return new FutureFailure(e); + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); } - return result.future; - } - private void checkRecordSize(int size) { - if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" - + " configured"); + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } } - } - - /** - * Flush any accumulated records from the producer. Blocks until all sends are complete. - */ - public void flush() { - log.debug("Flushing..."); - for (String topic : perTopicBatches.keySet()) { - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); - doSend(request, null, null); + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); } - } - - /** - * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change - * over time so this list should not be cached. - */ - public List partitionsFor(String topic) { - throw new NotImplementedException("Partitions not supported"); - } - - /** - * Return a map of metrics maintained by the producer - */ - public Map metrics() { - throw new NotImplementedException("Metrics not supported."); - } - - /** - * Close this producer - */ - public void close() { - close(0, null); - } - - /** - * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the - * timeout, fail any pending send requests and force close the producer. - */ - public void close(long timeout, TimeUnit unit) { - if (timeout < 0) { - throw new IllegalArgumentException("Timeout cannot be negative."); + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); } - channelUtil.closeChannel(); - log.debug("Closed producer"); - closed = true; - } + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } - public static class Builder { - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } - private PubsubChannelUtil channelUtil; - private PublisherFutureStub publisher; - private int batchSize; - private boolean isAcks; - private Map> perTopicBatches; - private int maxRequestSize; - private Time time; - - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { - Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); - this.project = project; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - setDefaults(); + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; } - private void setDefaults() { - // this is where to set 'regular' fields w/o side effects - this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; - this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; - this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; - this.time = new SystemTime(); + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); } - public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } - public Builder batchSize(int val) { - Preconditions.checkArgument(val > 0); - batchSize = val; - return this; + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); } - public Builder isAcks(boolean val) { isAcks = val; return this; } + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } - public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } - public Builder maxRequestSize(int val) { - Preconditions.checkArgument(val >= 0); - maxRequestSize = val; - return this; + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); } - public Builder time(Time val) { time = val; return this; } + private static class FutureFailure implements Future { - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + private final ExecutionException exception; - public PubsubProducer build() { - // this is where to set fields w/ side effects - if (channelUtil == null) { - this.channelUtil = new PubsubChannelUtil(); - } - if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); - } - return new PubsubProducer(this); - } - } + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ - private static class FutureFailure implements Future { - private final ExecutionException exception; + @Override + public boolean cancel(boolean interrupt) { + return false; + } - public FutureFailure(Exception e) { - this.exception = new ExecutionException(e); - } + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } - public boolean cancel(boolean interrupt) { - return false; - } + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + @Override + public boolean isCancelled() { + return false; + } - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + @Override + public boolean isDone() { + return true; + } - public boolean isCancelled() { - return false; } - public boolean isDone() { - return true; + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } } - } } diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index de8c1ae6..fd8e96b4 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,111 +14,100 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.pubsub.clients.producer; +package com.google.kafka.clients.producer; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.powermock.api.easymock.PowerMock.createMock; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; -import java.io.IOException; -import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.Before; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; -//@RunWith(PowerMockRunner.class) -//@PrepareForTest(PublisherFutureStub.class) -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - private static final String PROJECT = "unit-test-proj"; - private static final StringSerializer testSerializer = new StringSerializer(); - private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); - - private static PublisherFutureStub mockStub; - @Mock private static PubsubChannelUtil mockChannelUtil; - - private boolean channelIsClosed; - - - @Before - public void setUp() { - // mockStub = createMock(PublisherFutureStub.class); - MockitoAnnotations.initMocks(this); - channelIsClosed = false; - - Mockito.doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - return channelIsClosed = true; - } - }).when(mockChannelUtil).closeChannel(); - } - - /* send() tests */ - @Test - public void testSendPublisherClosed() throws IOException { - /* PubsubProducer testProducer = getNewProducer(); - testProducer.close(); - - try { - testProducer.send(testRecord); - fail("Should have thrown a runtime exception."); - } catch (RuntimeException expected) { - assertTrue("Channel should be closed.", channelIsClosed); - }*/ - } - - @Test - public void testSendRecordTooLarge() { - - } - - @Test - public void testSendBatchFull() { +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; - } - - /* This happens when the batch size is > 1 and not enough messages are batched */ - @Test - public void testSendMessageNotSentYet() { - - } - - /* flush() tests */ - @Test - public void testFlushMessagesSent() { - - } - - /* close() tests */ - @Test - public void testCloseTimeoutLessThanZero() { - - } - - @Test - public void testCloseChannelCloseSuccessful() { - - } - - private PubsubProducer getNewProducer() throws IOException { - - return new Builder<>(PROJECT, testSerializer, testSerializer) - .publisherFutureStub(mockStub) - .pubsubChannelUtil(mockChannelUtil) - .build(); - } +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } } diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 801b4d45b75804bef5139203762b7b1cff935269 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 088/140] scratching some of the mapped api to make it more pub/sub --- .../clients/consumer/PubsubConsumer.java | 119 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 5 files changed, 4 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index cadfbf1b..72eb4413 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -14,7 +14,7 @@ package com.google.kafka.cients.consumer; ======= -package com.google.kafka.clients.consumer; +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -162,121 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -329,7 +215,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From 9b31d7f1c9517ce7f4a48fe173624b99da532a25 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 089/140] Implementing publisher/producer using grpc backend. --- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../producer/PubsubProducerConfig.java | 5 +- .../com/google/pubsub/common/PubsubUtils.java | 46 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 9 files changed, 910 insertions(+), 1852 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 72eb4413..e5abef92 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,14 +1,16 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ <<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; @@ -16,18 +18,17 @@ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -36,7 +37,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -71,608 +71,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index f03c017f..66c08890 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -60,7 +60,10 @@ public class PubsubProducerConfig extends AbstractConfig { .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC) + .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From 04396cae075646d0e06eb5af5bedd1ec44336979 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 090/140] Add details to simplified producer --- .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../producer/PubsubProducerConfig.java | 3 +- .../internals/PubsubFutureRecordMetadata.java | 3 +- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 66c08890..170b1e03 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -63,7 +63,8 @@ public class PubsubProducerConfig extends AbstractConfig { .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC); + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 771fcb82..66d0603f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -22,6 +22,7 @@ import org.apache.kafka.clients.producer.RecordMetadata; public class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { return false; } @@ -41,4 +42,4 @@ public RecordMetadata get() throws InterruptedException, ExecutionException { public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } -} \ No newline at end of file +} From 09265159c0aff437692bc453e754b079f5e7a854 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 091/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 12 +++ .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 28 +++--- .../pubsub/clients/ProducerThreadPool.java | 23 +---- .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 1 - .../pubsub/common/PubsubChannelUtil.java | 21 ++--- .../com/google/pubsub/common/PubsubUtils.java | 46 --------- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 9 files changed, 90 insertions(+), 260 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index d9e7639a..515022b5 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,6 +23,18 @@ junit 4.12 + + org.powermock + powermock-module-junit4-legacy + 1.7.0RC2 + test + + + org.powermock + powermock-api-easymock + 1.7.0RC2 + test + org.apache.kafka kafka_2.10 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index d1ae43fa..b9aa6f54 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,14 +1,12 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; @@ -36,21 +34,19 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); - for (int i = 0; i < 1; i++) { - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); - } - } + ProducerRecord msg = new ProducerRecord(topic, "message" + command); + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); } - ); - } + } + } + ); Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index edb9707b..3c1447ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -16,7 +16,8 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) throws IOException { + + public static void main(String[] args) { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -25,7 +26,6 @@ public void uncaughtException(Thread t, Throwable e) { } }); - //ExecutorService executor = new ThreadPoolExecutor(1, 100, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), threadFactoryBuilder.build()); ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); Properties props = new Properties(); @@ -34,25 +34,12 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", "1") - .put("linger.ms", "1") + .put("batch.size", 1) + .put("linger.ms", 1) .build() ); - /* props.putAll(new ImmutableMap.Builder<>() - .put("max.block.ms", "30000") - //.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - //.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("bootstrap.servers", "104.198.72.101:9092") - .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB - // 10M, high enough to allow for duration to control batching - .put("batch.size", Integer.toString(10 * 1000 * 1000)) - .put("linger.ms", 10) - .build() - ); */ - - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 20; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..bcf86059 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null()); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 66d0603f..4c16e71f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -22,7 +22,6 @@ import org.apache.kafka.clients.producer.RecordMetadata; public class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { return false; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index a6d14269..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -19,18 +19,19 @@ import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; +import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.concurrent.Executors; -/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { - private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); + private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -40,16 +41,8 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; - /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() { - GoogleCredentials credentials; - try { - credentials = GoogleCredentials.getApplicationDefault() - .createScoped(CPS_SCOPE); - } catch (IOException exception) { - log.error("Exception occurred: " + exception.getMessage()); - return; - } + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java deleted file mode 100644 index 10a5599e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.pubsub.common; - -import com.google.auth.oauth2.GoogleCredentials; -import io.grpc.Channel; -import io.grpc.ClientInterceptors; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; -import io.grpc.netty.NegotiationType; -import io.grpc.netty.NettyChannelBuilder; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.Executors; - -public class PubsubUtils { - - private static final String ENDPOINT = "pubsub.googleapis.com"; - private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); - - public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; - public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From 5f0e17928692e89f44788b7c13c27c6caaf94846 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 092/140] Working on unit tests, need to mock the publisher calls --- pubsub-mapped-api/pom.xml | 14 +---- .../google/pubsub/clients/ProducerThread.java | 26 ++++---- .../pubsub/clients/ProducerThreadPool.java | 4 +- .../clients/producer/PubsubProducer.java | 30 ++------- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- 6 files changed, 75 insertions(+), 74 deletions(-) diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 515022b5..9dbf6b74 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -23,18 +23,6 @@ junit 4.12 - - org.powermock - powermock-module-junit4-legacy - 1.7.0RC2 - test - - - org.powermock - powermock-api-easymock - 1.7.0RC2 - test - org.apache.kafka kafka_2.10 @@ -98,7 +86,7 @@ org.mockito mockito-all - 1.9.5 + 2.0.2-beta diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index b9aa6f54..65bfe1d4 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -34,19 +34,21 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord(topic, "message" + command); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { + producer.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message"); + } + } } - } - } - ); + ); + } Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 3c1447ee..e9eda70b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -34,12 +34,12 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 1) + .put("batch.size", 20) .put("linger.ms", 1) .build() ); - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 1; i++) { Runnable worker = new ProducerThread("" + i, props, args[0]); executor.execute(worker); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index bcf86059..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -273,7 +251,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null()); + doSend(request, null, null); } } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} From 9895990d6d8ffd2f9e0bdcf66f87ea3fc7883944 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 093/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 1 + .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../producer/PubsubProducerConfig.java | 8 +- .../pubsub/common/PubsubChannelUtil.java | 4 - .../clients/producer/PubsubProducerTest.java | 35 +----- 5 files changed, 98 insertions(+), 50 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 65bfe1d4..8603a184 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,6 +1,7 @@ package com.google.pubsub.clients; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 170b1e03..fc20f034 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -59,12 +59,8 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, 1, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, 1*1024*1024, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(ACKS_CONFIG, Type.STRING, DEFAULT_ACKS, Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From 76f36514aad603540d167101a6a09f51a320de92 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 094/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 4 ++-- .../pubsub/clients/ProducerThreadPool.java | 7 +++---- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 8603a184..4f3e3427 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -35,8 +35,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index e9eda70b..36ef8716 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -16,8 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -34,8 +33,8 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", 20) - .put("linger.ms", 1) + .put("batch.size", "1") + .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From da2155126b52f0d90b35a20361ca8931a6e17fbe Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:46:08 -0800 Subject: [PATCH 095/140] Minor fixes to producer's classes --- .../pubsub/clients/consumer/PubsubConsumer.java | 3 --- .../pubsub/clients/producer/PubsubProducer.java | 2 +- .../com/google/pubsub/common/PubsubChannelUtil.java | 13 +++++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index e5abef92..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -12,9 +12,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ -<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java -package com.google.kafka.cients.consumer; -======= package com.google.pubsub.clients.consumer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..e9ca5753 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -360,7 +360,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..9088d5bf 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +44,14 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } From 2cb74efd8809a2dc3c12864f1ff9ec320a2532ac Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 10:06:42 -0800 Subject: [PATCH 096/140] Omitting unit tests and unused classes --- .../clients/producer/PubsubProducerTest.java | 60 ------------------- .../pubsub/common/PubsubChannelUtilTest.java | 51 ---------------- 2 files changed, 111 deletions(-) delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index 32555be2..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - - /* send() tests */ - @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub - - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal - } - - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); - - return new PubsubProducer(props); - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java deleted file mode 100644 index 2c5ad730..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.pubsub.common; - -import static org.junit.Assert.assertNotNull; - -import java.io.IOException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PubsubChannelUtilTest { - - private static PubsubChannelUtil channelUtil; - - @BeforeClass - public static void setUp() throws IOException { - channelUtil = new PubsubChannelUtil(); - } - - @AfterClass - public static void tearDown() { - channelUtil.closeChannel(); - } - - @Test - public void testGetCallCredentials() throws IOException { - assertNotNull(channelUtil.callCredentials()); - channelUtil.closeChannel(); - } - - @Test - public void testGetChannel() { - assertNotNull(channelUtil.channel()); - } -} From f079caf858df352175b601a83b8d2f60e04c3e4e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 16:19:17 -0800 Subject: [PATCH 097/140] Adding the mapped publisher to the loadtest framework. --- .../gce/mapped-publisher_startup_script.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh diff --git a/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh b/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh new file mode 100644 index 00000000..0b0631b5 --- /dev/null +++ b/load-test-framework/src/main/resources/gce/mapped-publisher_startup_script.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +####################################### +# Query GCE for a provided metadata field. +# See https://developers.google.com/compute/docs/metadata +# Globals: +# None +# Arguments: +# $1: The path to the metadata field to retrieve +# Returns: +# The value stored at the metadata field +####################################### +function metadata() { + curl --silent --show-error --header 'Metadata-Flavor: Google' \ + "http://metadata/computeMetadata/v1/${1}"; +} + +readonly TMP="$(mktemp -d)" +readonly BUCKET=$(metadata instance/attributes/bucket) + +[[ "${TMP}" != "" ]] || error mktemp failed + +# Download the loadtest binary to this machine and install Java 8. +/usr/bin/apt-get update +/usr/bin/apt-get install -y openjdk-8-jre-headless & PIDAPT=$! +/usr/bin/gsutil cp "gs://${BUCKET}/driver.jar" "${TMP}" + +wait $PIDAPT + +# Run the loadtest binary +java -Xmx5000M -cp ${TMP}/driver.jar com.google.pubsub.clients.mapped.MappedPublisherTask From 350e5344d795b3f37f7e280cea665b3164fa2611 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 7 Mar 2017 11:06:36 -0800 Subject: [PATCH 098/140] Attempting to fix 'cannot load' error for publisher task. --- ...appedPublisherTask.java => CPSPublisherTask.java} | 11 +++++++---- .../src/main/java/com/google/pubsub/flic/Driver.java | 12 ++++++------ .../com/google/pubsub/flic/controllers/Client.java | 11 ++++++----- .../com/google/pubsub/flic/output/SheetsService.java | 3 ++- ...h => cps-mapped-java-publisher_startup_script.sh} | 4 ++-- .../google/pubsub/flic/output/SheetsServiceTest.java | 5 +---- 6 files changed, 24 insertions(+), 22 deletions(-) rename load-test-framework/src/main/java/com/google/pubsub/clients/mapped/{MappedPublisherTask.java => CPSPublisherTask.java} (89%) rename load-test-framework/src/main/resources/gce/{mapped-publisher_startup_script.sh => cps-mapped-java-publisher_startup_script.sh} (95%) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java similarity index 89% rename from load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java rename to load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index ffb01012..e100f069 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/MappedPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -18,16 +18,16 @@ * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer * interface */ -public class MappedPublisherTask extends Task { +public class CPSPublisherTask extends Task { - private static final Logger log = LoggerFactory.getLogger(MappedPublisherTask.class); + private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); private final String topic; private final String payload; private final int batchSize; private final PubsubProducer publisher; @SuppressWarnings("unchecked") - private MappedPublisherTask(StartRequest request) { + private CPSPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); this.payload = LoadTestRunner.createMessage(request.getMessageSize()); @@ -43,7 +43,7 @@ private MappedPublisherTask(StartRequest request) { public static void main(String[] args) throws Exception { LoadTestRunner.Options options = new LoadTestRunner.Options(); new JCommander(options, args); - LoadTestRunner.run(options, MappedPublisherTask::new); + LoadTestRunner.run(options, CPSPublisherTask::new); } @Override @@ -66,4 +66,7 @@ public ListenableFuture doRun() { } return result; } + + @Override + public void shutdown() { publisher.close(); } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 3fa0803f..6bcfc600 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -131,11 +131,11 @@ public class Driver { private int kafkaSubscriberCount = 0; @Parameter( - names = {"--mapped_publisher_count"}, - description = "Number of mapped publishers to start." + names = {"--cps_mapped_publisher_count"}, + description = "Number of cps mapped publishers to start." ) - private int mappedPublisherCount = 0; + private int cpsMappedPublisherCount = 0; @Parameter( names = {"--message_size", "-m"}, @@ -379,9 +379,9 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } - if (mappedPublisherCount > 0) { + if (cpsMappedPublisherCount > 0) { clientParamsMap.put( - new ClientParams(ClientType.MAPPED_PUBLISHER, null), mappedPublisherCount); + new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), cpsMappedPublisherCount); } if (kafkaPublisherCount > 0) { clientParamsMap.put( @@ -413,7 +413,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + mappedPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index 68f34b47..523a0cb6 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -116,7 +116,7 @@ public static String getTopicSuffix(ClientType clientType) { case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return "mapped"; } return null; @@ -198,7 +198,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -305,7 +305,7 @@ public enum ClientType { CPS_VTK_JAVA_PUBLISHER, KAFKA_PUBLISHER, KAFKA_SUBSCRIBER, - MAPPED_PUBLISHER; + CPS_MAPPED_JAVA_PUBLISHER; public boolean isCpsPublisher() { switch (this) { @@ -313,6 +313,7 @@ public boolean isCpsPublisher() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -335,7 +336,7 @@ public boolean isPublisher() { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: case KAFKA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -349,7 +350,7 @@ public ClientType getSubscriberType() { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_VTK_JAVA_PUBLISHER: - case MAPPED_PUBLISHER: + case CPS_MAPPED_JAVA_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 08718d2c..a60d527a 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -78,6 +78,7 @@ private void fillClientCounts(Map> types) { cpsPublisherCount += (countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) != null) ? countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) : 0; cpsPublisherCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER): 0; cpsPublisherCount += (countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) : 0; + cpsPublisherCount += (countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) : 0; cpsSubscriberCount += (countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER): 0; cpsSubscriberCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER): 0; kafkaPublisherCount += (countMap.get(ClientType.KAFKA_PUBLISHER) != null) ? countMap.get(ClientType.KAFKA_PUBLISHER) : 0; @@ -170,7 +171,7 @@ public List>> getValuesList(Map> types = new HashMap<>(); int expectedCpsCount = 0; int expectedKafkaCount = 0; - int expectedMappedCount = 0; Map paramsMap = new HashMap<>(); for (ClientType type : ClientType.values()) { paramsMap.put(new ClientParams(type, ""), 1); @@ -45,10 +44,8 @@ public void testClientSwitch() { expectedCpsCount++; } else if (type.toString().startsWith("kafka")) { expectedKafkaCount++; - } else if (type.toString().startsWith("mapped")) { - expectedMappedCount++; } else { - fail("ClientType toString didn't start with cps, mapped, or kafka"); + fail("ClientType toString didn't start with cps or kafka"); } } types.put("zone-test", paramsMap); From eb10f7304c0e0f00ef0a4e7937c70d2ec99b167b Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 10:37:54 -0800 Subject: [PATCH 099/140] Fixed naming for cps_mapped_java_publisher --- .../src/main/java/com/google/pubsub/flic/Driver.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 6bcfc600..e1daa3c0 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -131,11 +131,11 @@ public class Driver { private int kafkaSubscriberCount = 0; @Parameter( - names = {"--cps_mapped_publisher_count"}, + names = {"--cps_mapped_java_publisher_count"}, description = "Number of cps mapped publishers to start." ) - private int cpsMappedPublisherCount = 0; + private int cpsMappedJavaPublisherCount = 0; @Parameter( names = {"--message_size", "-m"}, @@ -379,9 +379,10 @@ public void run(BiFunction, Controller> contr clientParamsMap.put( new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); } - if (cpsMappedPublisherCount > 0) { + if (cpsMappedJavaPublisherCount > 0) { clientParamsMap.put( - new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), cpsMappedPublisherCount); + new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), + cpsMappedJavaPublisherCount); } if (kafkaPublisherCount > 0) { clientParamsMap.put( @@ -413,7 +414,7 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedPublisherCount + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedJavaPublisherCount > 0, "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( From f65fdd99a9f3ce34edfbd95ceec90b3a148c24fd Mon Sep 17 00:00:00 2001 From: jrheizelman Date: Fri, 9 Dec 2016 15:25:51 -0800 Subject: [PATCH 100/140] Added initial classes and set-up for Kafka mapped api --- .../clients/producer/PubsubProducer.java | 656 ++++++++++++++++++ .../producer/internals/PubsubAccumulator.java | 546 +++++++++++++++ .../producer/internals/PubsubBatch.java | 181 +++++ .../producer/internals/PubsubSender.java | 524 ++++++++++++++ .../clients/producer/PubsubProducerTest.java | 113 +++ .../producer/internals/MockPubsubServer.java | 67 ++ .../internals/PubsubAccumulatorTest.java | 412 +++++++++++ .../producer/internals/PubsubSenderTest.java | 210 ++++++ 8 files changed, 2709 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java create mode 100644 mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java new file mode 100644 index 00000000..2077a3c1 --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java @@ -0,0 +1,656 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketOptions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class PubsubProducer implements Producer { + + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); + } + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); + } else { + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + } + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); + } + + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } + } + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); + } + + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } + + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } + + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; + } + + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); + } + + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } + + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } + + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); + } + + private static class FutureFailure implements Future { + + private final ExecutionException exception; + + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } + + @Override + public boolean cancel(boolean interrupt) { + return false; + } + + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + } + + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java new file mode 100644 index 00000000..e8ff1bba --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.CopyOnWriteMap; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} + * instances to be sent to the server. + *

+ * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless + * this behavior is explicitly disabled. + */ +public final class PubsubAccumulator { + private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); + + private int mutedcalls = 0; + private volatile boolean closed; + private final AtomicInteger flushesInProgress; + private final AtomicInteger appendsInProgress; + private final int batchSize; + private final CompressionType compression; + private final long lingerMs; + private final long retryBackoffMs; + private final BufferPool free; + private final Time time; + private final ConcurrentMap> batches; + private final PubsubAccumulator.IncompleteBatches incomplete; + // The following variables are only accessed by the sender thread, so we don't need to protect them. + private final Set muted; + + /** + * Create a new record accumulator + * + * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances + * @param totalSize The maximum memory the record accumulator can use. + * @param compression The compression codec for the records + * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for + * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some + * latency for potentially better throughput due to more batching (and hence fewer, larger requests). + * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids + * exhausting all retries in a short period of time. + * @param metrics The metrics + * @param time The time instance to use + */ + public PubsubAccumulator(int batchSize, + long totalSize, + CompressionType compression, + long lingerMs, + long retryBackoffMs, + Metrics metrics, + Time time) { + this.closed = false; + this.flushesInProgress = new AtomicInteger(0); + this.appendsInProgress = new AtomicInteger(0); + this.batchSize = batchSize; + this.compression = compression; + this.lingerMs = lingerMs; + this.retryBackoffMs = retryBackoffMs; + this.batches = new CopyOnWriteMap<>(); + String metricGrpName = "producer-metrics"; + this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); + this.incomplete = new PubsubAccumulator.IncompleteBatches(); + this.muted = new HashSet<>(); + this.time = time; + registerMetrics(metrics, metricGrpName); + } + + private void registerMetrics(Metrics metrics, String metricGrpName) { + MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); + Measurable waitingThreads = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.queued(); + } + }; + metrics.addMetric(metricName, waitingThreads); + + metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); + Measurable totalBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.totalMemory(); + } + }; + metrics.addMetric(metricName, totalBytes); + + metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); + Measurable availableBytes = new Measurable() { + public double measure(MetricConfig config, long now) { + return free.availableMemory(); + } + }; + metrics.addMetric(metricName, availableBytes); + + Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); + metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); + bufferExhaustedRecordSensor.add(metricName, new Rate()); + } + + /** + * Add a record to the accumulator, return the append result + *

+ * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created + *

+ * + * @param timestamp The timestamp of the record + * @param key The key for the record + * @param value The value for the record + * @param callback The user-supplied callback to execute when the request is complete + * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available + */ + public PubsubAccumulator.RecordAppendResult append(String topic, + long timestamp, + byte[] key, + byte[] value, + Callback callback, + long maxTimeToBlock) throws InterruptedException { + // We keep track of the number of appending thread to make sure we do not miss batches in + // abortIncompleteBatches(). + appendsInProgress.incrementAndGet(); + try { + Deque deque = getOrCreateDeque(topic); + synchronized (deque) { + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + return appendResult; + } + } + + // we don't have an in-progress record batch try to allocate a new batch + int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); + log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); + ByteBuffer buffer = free.allocate(size, maxTimeToBlock); + synchronized (deque) { + // Need to check if producer is closed again after grabbing the dequeue lock. + if (closed) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... + free.deallocate(buffer); + return appendResult; + } + MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); + PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); + FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); + + deque.addLast(batch); + incomplete.add(batch); + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); + } + } finally { + appendsInProgress.decrementAndGet(); + } + } + + /** + * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary + * resources (like compression streams buffers). + */ + private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, + Deque deque) { + PubsubBatch last = deque.peekLast(); + if (last != null) { + FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); + if (future == null) + last.records.close(); + else + return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); + } + return null; + } + + /** + * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout + * due to metadata being unavailable + */ + public List abortExpiredBatches(int requestTimeout, long now) { + List expiredBatches = new ArrayList<>(); + int count = 0; + for (Map.Entry> entry : this.batches.entrySet()) { + Deque dq = entry.getValue(); + String topic = entry.getKey(); + // We only check if the batch should be expired if the partition does not have a batch in flight. + // This is to prevent later batches from being expired while an earlier batch is still in progress. + // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection + // is only active in this case. Otherwise the expiration order is not guaranteed. + if (!muted.contains(topic)) { + synchronized (dq) { + // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut + PubsubBatch lastBatch = dq.peekLast(); + Iterator batchIterator = dq.iterator(); + while (batchIterator.hasNext()) { + PubsubBatch batch = batchIterator.next(); + boolean isFull = batch != lastBatch || batch.records.isFull(); + // check if the batch is expired + if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { + expiredBatches.add(batch); + count++; + batchIterator.remove(); + deallocate(batch); + } else { + // Stop at the first batch that has not expired. + break; + } + } + } + } + } + if (!expiredBatches.isEmpty()) + log.trace("Expired {} batches in accumulator", count); + + return expiredBatches; + } + + /** + * Re-enqueue the given record batch in the accumulator to retry + */ + public void reenqueue(PubsubBatch batch, long now) { + batch.attempts++; + batch.lastAttemptMs = now; + batch.lastAppendTime = now; + batch.setRetry(); + Deque deque = getOrCreateDeque(batch.topic); + synchronized (deque) { + deque.addFirst(batch); + } + } + + /** + * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable + * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated + * partition batches. + *

+ * A destination node is ready to send data if: + *

    + *
  1. There is at least one partition that is not backing off its send + *
  2. and those partitions are not muted (to prevent reordering if + * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} + * is set to one)
    3. + *
  3. and any of the following are true
    4. + *
      + *
    • The record set is full
      • + *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • + *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions + * are immediately considered ready).
      • + *
    • The accumulator has been closed
      • + *
    + *
+ */ + public Set ready(long nowMs) { + Set readyTopics = new HashSet<>(); + + boolean exhausted = this.free.queued() > 0; + for (Map.Entry> entry : this.batches.entrySet()) { + String topic = entry.getKey(); + Deque deque = entry.getValue(); + + synchronized (deque) { + if (!muted.contains(topic)) { + PubsubBatch batch = deque.peekFirst(); + if (batch != null) { + boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; + long waitedTimeMs = nowMs - batch.lastAttemptMs; + long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; + long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); + boolean full = deque.size() > 1 || batch.records.isFull(); + boolean expired = waitedTimeMs >= timeToWaitMs; + boolean sendable = full || expired || exhausted || closed || flushInProgress(); + if (sendable && !backingOff) { + readyTopics.add(topic); + } + } + } + } + } + + return readyTopics; + } + + /** + * @return Whether there is any unsent record in the accumulator. + */ + public boolean hasUnsent() { + for (Map.Entry> entry : this.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + if (!deque.isEmpty()) + return true; + } + } + return false; + } + + /** + * Drain all the data and collates it into a list of batches that will fit within the specified size. + * + * @param maxSize The maximum number of bytes to drain + * @param now The current unix time in milliseconds + * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. + */ + public Map> drain(Set topics, int maxSize, long now) { + if (topics.isEmpty()) { + return Collections.emptyMap(); + } + Map> out = new HashMap<>(); + for (String topic : topics) { + int size = 0; + List ready = new ArrayList<>(); + out.put(topic, ready); + if (muted.contains(topic)) { + continue; + } + Deque deque = getDeque(topic); + synchronized (deque) { + PubsubBatch first = deque.peekFirst(); + if (first != null) { + boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; + // Only drain the batch if it is not during backoff period. + if (!backoff) { + if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { + PubsubBatch batch = deque.pollFirst(); + batch.records.close(); + size += batch.records.sizeInBytes(); + ready.add(batch); + batch.drainedMs = now; + } + } + } + } + } + return out; + } + + private Deque getDeque(String topic) { + return batches.get(topic); + } + + /** + * Get the deque for the given topic-partition, creating it if necessary. + */ + private Deque getOrCreateDeque(String topic) { + Deque d = this.batches.get(topic); + if (d != null) + return d; + d = new ArrayDeque<>(); + Deque previous = this.batches.putIfAbsent(topic, d); + if (previous == null) + return d; + else + return previous; + } + + /** + * Deallocate the record batch + */ + public void deallocate(PubsubBatch batch) { + incomplete.remove(batch); + free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); + } + + /** + * Are there any threads currently waiting on a flush? + * + * package private for test + */ + boolean flushInProgress() { + return flushesInProgress.get() > 0; + } + + /* Visible for testing */ + Map> batches() { + return Collections.unmodifiableMap(batches); + } + + /** + * Initiate the flushing of data from the accumulator...this makes all requests immediately ready + */ + public void beginFlush() { + this.flushesInProgress.getAndIncrement(); + } + + /** + * Are there any threads currently appending messages? + */ + private boolean appendsInProgress() { + return appendsInProgress.get() > 0; + } + + /** + * Mark all partitions as ready to send and block until the send is complete + */ + public void awaitFlushCompletion() throws InterruptedException { + try { + for (PubsubBatch batch : this.incomplete.all()) + batch.produceFuture.await(); + } finally { + this.flushesInProgress.decrementAndGet(); + } + } + + /** + * This function is only called when sender is closed forcefully. It will fail all the + * incomplete batches and return. + */ + public void abortIncompleteBatches() { + // We need to keep aborting the incomplete batch until no thread is trying to append to + // 1. Avoid losing batches. + // 2. Free up memory in case appending threads are blocked on buffer full. + // This is a tight loop but should be able to get through very quickly. + do { + abortBatches(); + } while (appendsInProgress()); + // After this point, no thread will append any messages because they will see the close + // flag set. We need to do the last abort after no thread was appending in case there was a new + // batch appended by the last appending thread. + abortBatches(); + this.batches.clear(); + } + + /** + * Go through incomplete batches and abort them. + */ + private void abortBatches() { + for (PubsubBatch batch : incomplete.all()) { + Deque deque = getDeque(batch.topic); + // Close the batch before aborting + synchronized (deque) { + batch.records.close(); + deque.remove(batch); + } + batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); + deallocate(batch); + } + } + + public void muteTopic(String topic) { + mutedcalls++; + muted.add(topic); + } + + public void unmuteTopic(String topic) { + mutedcalls++; + muted.remove(topic); + } + + public boolean isMutedTopic(String topic) { + return muted.contains(topic); + } + + /** + * Close this accumulator and force all the record buffers to be drained + */ + public void close() { + this.closed = true; + } + + /* + * Metadata about a record just appended to the record accumulator + */ + public final static class RecordAppendResult { + public final FutureRecordMetadata future; + public final boolean batchIsFull; + public final boolean newBatchCreated; + + public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { + this.future = future; + this.batchIsFull = batchIsFull; + this.newBatchCreated = newBatchCreated; + } + } + + /* + * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet + */ + private final static class IncompleteBatches { + private final Set incomplete; + + public IncompleteBatches() { + this.incomplete = new HashSet(); + } + + public void add(PubsubBatch batch) { + synchronized (incomplete) { + this.incomplete.add(batch); + } + } + + public void remove(PubsubBatch batch) { + synchronized (incomplete) { + boolean removed = this.incomplete.remove(batch); + if (!removed) + throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); + } + } + + public Iterable all() { + synchronized (incomplete) { + return new ArrayList<>(this.incomplete); + } + } + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java new file mode 100644 index 00000000..6f60d8fd --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A batch of records that is or will be sent. + * + * This class is not thread safe and external synchronization must be used when modifying it + */ +public final class PubsubBatch { + + private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); + + public int recordCount = 0; + public int maxRecordSize = 0; + public volatile int attempts = 0; + public final long createdMs; + public long drainedMs; + public long lastAttemptMs; + public final MemoryRecords records; + public final ProduceRequestResult produceFuture; + public long lastAppendTime; + public String topic; + + private final List thunks; + private long offsetCounter = 0L; + private boolean retry; + + public PubsubBatch(String topic, MemoryRecords records, long now) { + this.createdMs = now; + this.lastAttemptMs = now; + this.records = records; + this.produceFuture = new ProduceRequestResult(); + this.thunks = new ArrayList(); + this.lastAppendTime = createdMs; + this.retry = false; + this.topic = topic; + } + + /** + * Append the record to the current record set and return the relative offset within that record set + * + * @return The RecordSend corresponding to this record or null if there isn't sufficient room. + */ + public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { + if (!this.records.hasRoomFor(key, value)) { + return null; + } else { + long checksum = this.records.append(offsetCounter++, timestamp, key, value); + this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); + this.lastAppendTime = now; + FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, + timestamp, checksum, + key == null ? -1 : key.length, + value == null ? -1 : value.length); + if (callback != null) + thunks.add(new Thunk(callback, future)); + this.recordCount++; + return future; + } + } + + /** + * Complete the request + * + * @param baseOffset The base offset of the messages assigned by the server + * @param timestamp The timestamp returned by the broker. + * @param exception The exception that occurred (or null if the request was successful) + */ + public void done(long baseOffset, long timestamp, RuntimeException exception) { + TopicPartition tp = new TopicPartition(topic, 0); + log.trace("Produced messages with base offset offset {} and error: {}.", + baseOffset, + exception); + // execute callbacks + for (int i = 0; i < this.thunks.size(); i++) { + try { + Thunk thunk = this.thunks.get(i); + if (exception == null) { + // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. + RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), + timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, + thunk.future.checksum(), + thunk.future.serializedKeySize(), + thunk.future.serializedValueSize()); + thunk.callback.onCompletion(metadata, null); + } else { + thunk.callback.onCompletion(null, exception); + } + } catch (Exception e) { + log.error("Error executing user-provided callback on message for topic {}:", topic, e); + } + } + this.produceFuture.done(tp, baseOffset, exception); + } + + /** + * A callback and the associated FutureRecordMetadata argument to pass to it. + */ + final private static class Thunk { + final Callback callback; + final FutureRecordMetadata future; + + public Thunk(Callback callback, FutureRecordMetadata future) { + this.callback = callback; + this.future = future; + } + } + + @Override + public String toString() { + return "RecordBatch(recordCount=" + recordCount + ")"; + } + + /** + * A batch whose metadata is not available should be expired if one of the following is true: + *
    + *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). + *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. + *
+ */ + public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { + boolean expire = false; + String errorMessage = null; + + if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { + expire = true; + errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; + } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { + expire = true; + errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; + } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { + expire = true; + errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; + } + + if (expire) { + this.records.close(); + this.done(-1L, Record.NO_TIMESTAMP, + new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); + } + + return expire; + } + + /** + * Returns if the batch is been retried for sending to kafka + */ + public boolean inRetry() { + return this.retry; + } + + /** + * Set retry to true if the batch is being retried (for send) + */ + public void setRetry() { + this.retry = true; + } +} diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java new file mode 100644 index 00000000..aaf0580e --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java @@ -0,0 +1,524 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PubsubMessage; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata + * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. + */ +public class PubsubSender implements Runnable { + private static final Logger log = LoggerFactory.getLogger(Sender.class); + + /* the record accumulator that batches records */ + private final PubsubAccumulator accumulator; + + /* the grpc stub to send records to pubsub */ + private final PublisherGrpc.PublisherFutureStub stub; + + private final ThreadPoolExecutor executor; + + /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ + private final boolean guaranteeMessageOrder; + + /* the maximum request size to attempt to send to the server */ + private final int maxRequestSize; + + /* the number of times to retry a failed request before giving up */ + private final int retries; + + /* the clock instance used for getting the time */ + private final Time time; + + /* true while the sender thread is still running */ + private volatile boolean running; + + /* true when the caller wants to ignore all unsent/inflight messages and force close. */ + private volatile boolean forceClose; + + /* metrics */ + private final PubsubSenderMetrics sensors; + + /* the max time to wait for the server to respond to the request*/ + private final int requestTimeout; + + public PubsubSender(ManagedChannel channel, + PubsubAccumulator accumulator, + boolean guaranteeMessageOrder, + int maxRequestSize, + int retries, + Metrics metrics, + Time time, + int requestTimeout) { + this.accumulator = accumulator; + this.guaranteeMessageOrder = guaranteeMessageOrder; + this.maxRequestSize = maxRequestSize; + this.running = true; + this.retries = retries; + this.time = time; + this.sensors = new PubsubSenderMetrics(metrics); + this.requestTimeout = requestTimeout; + this.stub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); + this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue()); + } + + @Override + public void run() { + log.debug("Starting Kafka producer I/O thread."); + + // main loop, runs until close is called + while (running) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + + log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); + + // okay we stopped accepting requests but there may still be + // requests in the accumulator or waiting for acknowledgment, + // wait until these are completed. + while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { + try { + run(time.milliseconds()); + } catch (Exception e) { + log.error("Uncaught error in kafka producer I/O thread: ", e); + } + } + if (forceClose) { + // We need to fail all the incomplete batches and wake up the threads waiting on + // the futures. + this.accumulator.abortIncompleteBatches(); + } + try { + this.executor.shutdown(); + } catch (Exception e) { + log.error("Failed to close network client", e); + } + + log.debug("Shutdown of Kafka producer I/O thread has completed."); + } + + /** + * Run a single iteration of sending + * + * @param now + * The current POSIX time in milliseconds + */ + void run(long now) { + Set readyTopics = this.accumulator.ready(now); + // create produce requests + Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); + if (guaranteeMessageOrder) { + // Mute all the partitions drained + for (List batchList : batches.values()) { + for (PubsubBatch batch : batchList) { + synchronized (accumulator) { + if (accumulator.isMutedTopic(batch.topic)) { + log.info("Another thread got same ordered topic before lock, removing.", batch.topic); + } else { + this.accumulator.muteTopic(batch.topic); + } + } + } + } + } + + List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); + // update sensors + for (PubsubBatch expiredBatch : expiredBatches) + this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); + + sensors.updateProduceRequestMetrics(batches); + + if (!readyTopics.isEmpty()) { + log.trace("Topics with data ready to send: {}", readyTopics); + } + sendProduceRequests(batches, now); + } + + /** + * Start closing the sender (won't actually complete until all data is sent out) + */ + public void initiateClose() { + // Ensure accumulator is closed first to guarantee that no more appends are accepted after + // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. + this.accumulator.close(); + this.running = false; + this.executor.shutdown(); + } + + /** + * Closes the sender without sending out any pending messages. + */ + public void forceClose() { + this.forceClose = true; + initiateClose(); + } + + /** + * Complete or retry the given batch of records. + * + * @param batch The record batch + * @param error The error (or null if none) + * @param baseOffset The base offset assigned to the records if successful + * @param timestamp The timestamp returned by the broker for this batch + */ + private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { + if (error != Errors.NONE && canRetry(batch, error)) { + // retry + log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", + batch.topic, + this.retries - batch.attempts - 1, + error); + batch.done(baseOffset, timestamp, error.exception()); + this.accumulator.reenqueue(batch, time.milliseconds()); + this.sensors.recordRetries(batch.topic, batch.recordCount); + } else { + RuntimeException exception; + if (error == Errors.TOPIC_AUTHORIZATION_FAILED) + exception = new TopicAuthorizationException(batch.topic); + else + exception = error.exception(); + // tell the user the result of their request + batch.done(baseOffset, timestamp, exception); + this.accumulator.deallocate(batch); + if (error != Errors.NONE) + this.sensors.recordErrors(batch.topic, batch.recordCount); + } + + // Unmute the completed partition. + if (guaranteeMessageOrder) + this.accumulator.unmuteTopic(batch.topic); + } + + /** + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + */ + private boolean canRetry(PubsubBatch batch, Errors error) { + return batch.attempts < this.retries && error.exception() instanceof RetriableException; + } + + /** + * Transfer the record batches into a list of produce requests on a per-node basis + */ + private void sendProduceRequests(Map> collated, long now) { + for (Map.Entry> entry : collated.entrySet()) + sendProduceRequest(now, requestTimeout, entry.getValue()); + } + + /** + * Create a produce request from the given record batches + */ + private void sendProduceRequest(long sendTime, long timeout, List batches) { + for (PubsubBatch batch : batches) { + PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); + request.addMessages(PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(batch.records.buffer()))); + executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); + log.trace("Sent produce request to topic {}", batch.topic); + } + } + + private class ProduceRequestThread implements Runnable { + private PublishRequest request; + private PubsubBatch batch; + private long timeout; + private long sendTime; + + public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { + this.timeout = timeout; + this.sendTime = sendTime; + this.request = request; + this.batch = batch; + } + + @Override + public void run() { + long receivedTime = time.milliseconds(); + ListenableFuture future = stub.publish(request); + while (true) { + try { + PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); + log.trace("Receved produce response from topic {}", batch.topic); + String id = response.getMessageIds(0); + long offset = Long.valueOf(id); + completeBatch(batch, Errors.NONE, offset, receivedTime); + return; + } catch (InterruptedException e) { + log.warn("Accessing publish future was interrupted, retrying"); + } catch (TimeoutException e) { + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + return; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof StatusRuntimeException) { + Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); + switch (code) { + case ABORTED: + case CANCELLED: + case INTERNAL: + completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); + break; + case DATA_LOSS: + completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); + break; + case DEADLINE_EXCEEDED: + completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); + break; + case ALREADY_EXISTS: + case OUT_OF_RANGE: + case INVALID_ARGUMENT: + completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); + break; + case NOT_FOUND: + completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); + break; + case RESOURCE_EXHAUSTED: + completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); + break; + case PERMISSION_DENIED: + case UNAUTHENTICATED: + completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); + break; + case FAILED_PRECONDITION: + case UNAVAILABLE: + completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); + break; + case UNIMPLEMENTED: + completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); + break; + default: + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + break; + } + } else { // Status is not StatusRuntimeException + completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); + } + return; + } + sensors.recordLatency(batch.topic, receivedTime - sendTime); + } + } + } + + private class PubsubSenderMetrics { + private final Metrics metrics; + public final Sensor retrySensor; + public final Sensor errorSensor; + public final Sensor queueTimeSensor; + public final Sensor requestTimeSensor; + public final Sensor recordsPerRequestSensor; + public final Sensor batchSizeSensor; + public final Sensor compressionRateSensor; + public final Sensor maxRecordSizeSensor; + public final Sensor produceThrottleTimeSensor; + + public PubsubSenderMetrics(Metrics metrics) { + this.metrics = metrics; + String metricGrpName = "producer-metrics"; + + this.batchSizeSensor = metrics.sensor("batch-size"); + MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Avg()); + m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); + this.batchSizeSensor.add(m, new Max()); + + this.compressionRateSensor = metrics.sensor("compression-rate"); + m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); + this.compressionRateSensor.add(m, new Avg()); + + this.queueTimeSensor = metrics.sensor("queue-time"); + m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Avg()); + m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); + this.queueTimeSensor.add(m, new Max()); + + this.requestTimeSensor = metrics.sensor("request-time"); + m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); + this.requestTimeSensor.add(m, new Avg()); + m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); + this.requestTimeSensor.add(m, new Max()); + + this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); + m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Avg()); + m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); + this.produceThrottleTimeSensor.add(m, new Max()); + + this.recordsPerRequestSensor = metrics.sensor("records-per-request"); + m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); + this.recordsPerRequestSensor.add(m, new Rate()); + m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); + this.recordsPerRequestSensor.add(m, new Avg()); + + this.retrySensor = metrics.sensor("record-retries"); + m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); + this.retrySensor.add(m, new Rate()); + + this.errorSensor = metrics.sensor("errors"); + m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); + this.errorSensor.add(m, new Rate()); + + this.maxRecordSizeSensor = metrics.sensor("record-size-max"); + m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); + this.maxRecordSizeSensor.add(m, new Max()); + m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); + this.maxRecordSizeSensor.add(m, new Avg()); + + m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); + this.metrics.addMetric(m, new Measurable() { + public double measure(MetricConfig config, long now) { + return executor.getActiveCount(); + } + }); + } + + private void maybeRegisterTopicMetrics(String topic) { + // if one sensor of the metrics has been registered for the topic, + // then all other sensors should have been registered; and vice versa + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); + if (topicRecordCount == null) { + Map metricTags = Collections.singletonMap("topic", topic); + String metricGrpName = "producer-topic-metrics"; + + topicRecordCount = this.metrics.sensor(topicRecordsCountName); + MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); + topicRecordCount.add(m, new Rate()); + + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = this.metrics.sensor(topicByteRateName); + m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); + topicByteRate.add(m, new Rate()); + + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); + m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); + topicCompressionRate.add(m, new Avg()); + + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); + m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); + topicRetrySensor.add(m, new Rate()); + + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); + m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); + topicErrorSensor.add(m, new Rate()); + } + } + + public void updateProduceRequestMetrics(Map> batches) { + long now = time.milliseconds(); + for (List topicBatch : batches.values()) { + int records = 0; + for (PubsubBatch batch : topicBatch) { + // register all per-topic metrics at once + String topic = batch.topic; + maybeRegisterTopicMetrics(topic); + + // per-topic record send rate + String topicRecordsCountName = "topic." + topic + ".records-per-batch"; + Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); + topicRecordCount.record(batch.recordCount); + + // per-topic bytes send rate + String topicByteRateName = "topic." + topic + ".bytes"; + Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); + topicByteRate.record(batch.records.sizeInBytes()); + + // per-topic compression rate + String topicCompressionRateName = "topic." + topic + ".compression-rate"; + Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); + topicCompressionRate.record(batch.records.compressionRate()); + + // global metrics + this.batchSizeSensor.record(batch.records.sizeInBytes(), now); + this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); + this.compressionRateSensor.record(batch.records.compressionRate()); + this.maxRecordSizeSensor.record(batch.maxRecordSize, now); + records += batch.recordCount; + } + this.recordsPerRequestSensor.record(records, now); + } + } + + public void recordRetries(String topic, int count) { + long now = time.milliseconds(); + this.retrySensor.record(count, now); + String topicRetryName = "topic." + topic + ".record-retries"; + Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); + if (topicRetrySensor != null) + topicRetrySensor.record(count, now); + } + + public void recordErrors(String topic, int count) { + long now = time.milliseconds(); + this.errorSensor.record(count, now); + String topicErrorName = "topic." + topic + ".record-errors"; + Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); + if (topicErrorSensor != null) + topicErrorSensor.record(count, now); + } + + public void recordLatency(String node, long latency) { + long now = time.milliseconds(); + this.requestTimeSensor.record(latency, now); + if (!node.isEmpty()) { + String nodeTimeName = "node-" + node + ".latency"; + Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); + if (nodeRequestTime != null) + nodeRequestTime.record(latency, now); + } + } + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..fd8e96b4 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.kafka.clients.producer; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.test.MockMetricsReporter; +import org.apache.kafka.test.MockProducerInterceptor; +import org.apache.kafka.test.MockSerializer; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") +public class PubsubProducerTest { + + @Test + public void testSerializerClose() throws Exception { + Map configs = new HashMap<>(); + configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); + final int oldInitCount = MockSerializer.INIT_COUNT.get(); + final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + + PubsubProducer producer = new PubsubProducer( + configs, new MockSerializer(), new MockSerializer()); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + + producer.close(); + Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); + Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); + } + + @Test + public void testInterceptorConstructClose() throws Exception { + try { + Properties props = new Properties(); + // test with client ID assigned by PubsubProducer + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); + props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); + + PubsubProducer producer = new PubsubProducer( + props, new StringSerializer(), new StringSerializer()); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); + + // Cluster metadata will only be updated on calling onSend. + Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); + + producer.close(); + Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); + Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); + } finally { + // cleanup since we are using mutable static variables in MockProducerInterceptor + MockProducerInterceptor.resetCounters(); + } + } + + @Test + public void testOsDefaultSocketBufferSizes() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); + PubsubProducer producer = new PubsubProducer<>( + config, new ByteArraySerializer(), new ByteArraySerializer()); + producer.close(); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketSendBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } + + @Test(expected = KafkaException.class) + public void testInvalidSocketReceiveBufferSize() throws Exception { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); + new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java new file mode 100644 index 00000000..a4b2ce53 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import io.grpc.stub.StreamObserver; +import java.util.LinkedList; +import java.util.Queue; + +public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { + private Queue> responseList; + + public MockPubsubServer() { + responseList = new LinkedList<>(); + } + + @Override + public void publish(PublishRequest request, StreamObserver responseObserver) { + responseList.add(responseObserver); + } + + public int inFlightCount() { + return responseList.size(); + } + + public void respond(PublishResponse response) { + StreamObserver stream = responseList.poll(); + stream.onNext(response); + stream.onCompleted(); + } + + public void disconnect() { + for (int i = 0; i < 100; i++) { + if (responseList.isEmpty()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { } // not an issue, ignore + } + } + StreamObserver stream = responseList.poll(); + stream.onCompleted(); + } + + public boolean listen(int messagesExpected, long waitInMillis) { + for (int i = 0; i < waitInMillis / 50; i++) { + if (responseList.size() == messagesExpected) { + return true; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + return false; + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java new file mode 100644 index 00000000..fe241b0c --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java @@ -0,0 +1,412 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.LogEntry; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.SystemTime; +import org.junit.After; +import org.junit.Test; + +public class PubsubAccumulatorTest { + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private SystemTime systemTime = new SystemTime(); + private byte[] key = "key".getBytes(); + private byte[] value = "value".getBytes(); + private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); + private Metrics metrics = new Metrics(time); + private final long maxBlockTimeMs = 1000; + + @After + public void teardown() { + this.metrics.close(); + } + + @Test + public void testFull() throws Exception { + long now = time.milliseconds(); + int batchSize = 1024; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = batchSize / msgSize; + for (int i = 0; i < appends; i++) { + // append to the first batch + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque batches = accum.batches().get(topic); + assertEquals(1, batches.size()); + assertTrue(batches.peekFirst().records.isWritable()); + assertEquals("No topics should be ready.", 0, accum.ready(now).size()); + } + + // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Deque allBatches = accum.batches().get(topic); + assertEquals(2, allBatches.size()); + Iterator batchesIterator = allBatches.iterator(); + assertFalse(batchesIterator.next().records.isWritable()); + assertTrue(batchesIterator.next().records.isWritable()); + assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + for (int i = 0; i < appends; i++) { + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + } + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testAppendLarge() throws Exception { + int batchSize = 512; + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); + assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + } + + @Test + public void testLinger() throws Exception { + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); + time.sleep(10); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); + assertEquals(1, batches.size()); + PubsubBatch batch = batches.get(0); + + Iterator iter = batch.records.iterator(); + LogEntry entry = iter.next(); + assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); + assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); + assertFalse("No more records", iter.hasNext()); + } + + @Test + public void testPartialDrain() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); + int appends = 1024 / msgSize + 1; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } + assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); + + List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); + assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); + } + + @SuppressWarnings("unused") + @Test + public void testStressfulSituation() throws Exception { + final int numThreads = 5; + final int msgs = 10000; + final int numParts = 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); + List threads = new ArrayList(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread() { + public void run() { + for (int i = 0; i < msgs; i++) { + try { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }); + } + for (Thread t : threads) + t.start(); + int read = 0; + long now = time.milliseconds(); + while (read < numThreads * msgs) { + Set readyTopics = accum.ready(now); + List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); + if (batches != null) { + for (PubsubBatch batch : batches) { + for (LogEntry entry : batch.records) + read++; + accum.deallocate(batch); + } + } + } + + for (Thread t : threads) + t.join(); + } + + @Test + public void testNextReadyCheckDelay() throws Exception { + // Next check time will use lingerMs since this test won't trigger any retries/backoff + long lingerMs = 10L; + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + // Just short of going over the limit so we trigger linger time + int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + time.sleep(lingerMs / 2); + + for (int i = 0; i < appends; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyNodes.size()); + + // Add enough to make data sendable immediately + for (int i = 0; i < appends + 1; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyNodes = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); + } + + @Test + public void testRetryBackoff() throws Exception { + long lingerMs = Long.MAX_VALUE / 4; + long retryBackoffMs = Long.MAX_VALUE / 2; + final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + + long now = time.milliseconds(); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + Set readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); + Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); + assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); + assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); + + // Reenqueue the batch + now = time.milliseconds(); + accum.reenqueue(batches.get(topic).get(0), now); + + // Put another message into accumulator + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + readyTopics = accum.ready(now + lingerMs + 1); + assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); + + // topic though backoff, should drain both batches + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); + readyTopics = accum.ready(now + retryBackoffMs + 1); + batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); + assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); + } + + @Test + public void testFlush() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.beginFlush(); + readyTopics = accum.ready(time.milliseconds()); + + // drain and deallocate all batches + Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + for (List batches: results.values()) + for (PubsubBatch batch: batches) + accum.deallocate(batch); + + // should be complete with no unsent records. + accum.awaitFlushCompletion(); + assertFalse(accum.hasUnsent()); + } + + private void delayedInterrupt(final Thread thread, final long delayMs) { + Thread t = new Thread() { + public void run() { + systemTime.sleep(delayMs); + thread.interrupt(); + } + }; + t.start(); + } + + @Test + public void testAwaitFlushComplete() throws Exception { + PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + + accum.beginFlush(); + assertTrue(accum.flushInProgress()); + delayedInterrupt(Thread.currentThread(), 1000L); + try { + accum.awaitFlushCompletion(); + fail("awaitFlushCompletion should throw InterruptException"); + } catch (InterruptedException e) { + assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); + } + } + + @Test + public void testAbortIncompleteBatches() throws Exception { + long lingerMs = Long.MAX_VALUE; + int batchSize = 4 * 1024; + final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); + final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); + class TestCallback implements Callback { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); + numExceptionReceivedInCallback.incrementAndGet(); + } + } + int attempts = batchSize / msgSize; + for (int i = 0; i < attempts; i++) + accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No topics should be ready.", 0, readyTopics.size()); + + accum.abortIncompleteBatches(); + assertEquals(numExceptionReceivedInCallback.get(), attempts); + assertFalse(accum.hasUnsent()); + + } + + @Test + public void testExpiredBatches() throws InterruptedException { + long retryBackoffMs = 100L; + long lingerMs = 3000L; + int batchSize = 1024; + int requestTimeout = 60; + + PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); + int appends = batchSize / msgSize; + + // Test batches not in retry + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + } + // Make the batches ready due to batch full + accum.append(topic, 0L, key, value, null, 0); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + // Advance the clock to expire the batch. + time.sleep(requestTimeout + 1); + accum.muteTopic(topic); + List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Advance the clock to make the next batch ready due to linger.ms + time.sleep(lingerMs); + assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); + time.sleep(requestTimeout + 1); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); + assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); + + // Test batches in retry. + // Create a retried batch + accum.append(topic, 0L, key, value, null, 0); + time.sleep(lingerMs); + readyTopics = accum.ready(time.milliseconds()); + assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("There should be only one batch.", drained.get(topic).size(), 1); + time.sleep(1000L); + accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); + + // test expiration. + time.sleep(requestTimeout + retryBackoffMs); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired.", 0, expiredBatches.size()); + time.sleep(1L); + + accum.muteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); + + accum.unmuteTopic(topic); + expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); + assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); + } + + @Test + public void testMutedPartitions() throws InterruptedException { + long now = time.milliseconds(); + PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); + int appends = 1024 / msgSize; + for (int i = 0; i < appends; i++) { + accum.append(topic, 0L, key, value, null, maxBlockTimeMs); + assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); + } + time.sleep(2000); + + // Test ready with muted partition + accum.muteTopic(topic); + Set readyTopics = accum.ready(time.milliseconds()); + assertEquals("No node should be ready", 0, readyTopics.size()); + + // Test ready without muted partition + accum.unmuteTopic(topic); + readyTopics = accum.ready(time.milliseconds()); + assertTrue("The batch should be ready", readyTopics.size() > 0); + + // Test drain with muted partition + accum.muteTopic(topic); + Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertEquals("No batch should have been drained", 0, drained.get(topic).size()); + + // Test drain without muted partition. + accum.unmuteTopic(topic); + drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); + assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); + } +} diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java new file mode 100644 index 00000000..15256379 --- /dev/null +++ b/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.producer.internals; + +import com.google.pubsub.v1.PublishResponse; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.MockTime; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PubsubSenderTest { + + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final String METRIC_GROUP = "producer-metrics"; + private static final double EPS = 0.0001; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 10000; + + private String topic = "test-topic"; + private MockTime time = new MockTime(); + private int batchSize = 16 * 1024; + private Metrics metrics = null; + private PubsubAccumulator accumulator = null; + + @Rule + public Timeout globalTimeout = Timeout.seconds(15); + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + metrics = new Metrics(metricConfig, time); + accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); + } + + @After + public void tearDown() { + this.metrics.close(); + } + + @Test + public void testSimple() throws Exception { + MockPubsubServer server = newServer("testSimple"); + PubsubSender sender = newSender("testSimple", MAX_RETRIES); + long offset = 32; + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // Sends produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + assertNotNull("Request should be completed", future.get()); + waitForUnmute(topic, 1000); + } + + @Test + public void testRetries() throws Exception { + int maxRetries = 1; + MockPubsubServer server = newServer("testRetries"); + PubsubSender sender = newSender("testRetries", maxRetries); + // do a successful retry + Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + assertEquals("All requests completed.", 0, server.inFlightCount()); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + sender.run(time.milliseconds()); // send second produce request + assertTrue("Server should receive request..", server.listen(1, 1000)); + long offset = 32; + server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); + assertEquals("All requests completed.", 0, (long) server.inFlightCount()); + eventualReturn(future, 1000); + assertEquals(offset, future.get().offset()); + waitForUnmute(topic, 1000); + + // do an unsuccessful retry + future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + for (int i = 0; i < maxRetries + 1; i++) { + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + server.disconnect(); + completedWithError(future, Errors.NETWORK_EXCEPTION); + waitForUnmute(topic, 1000); + } + sender.run(time.milliseconds()); + assertEquals("Retry request should be received.", 0, server.inFlightCount()); + waitForUnmute(topic, 1000); + } + + @Test + public void testSendInOrder() throws Exception { + PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); + MockPubsubServer server = newServer("testSendInOrder"); + + // Send the first message. + accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); // send produce request + assertTrue("Server should receive request.", server.listen(1, 1000)); + + time.sleep(900); + // Now send another message + accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); + + // Sender should not send second message before first is returned + sender.run(time.milliseconds()); + assertTrue("Server expects only one request.", server.listen(1, 1000)); + } + + private void completedWithError(Future future, Errors error) throws Exception { + try { + future.get(); + fail("Should have thrown an exception."); + } catch (ExecutionException e) { + assertEquals(error.exception().getClass(), e.getCause().getClass()); + } + } + + private void eventualReturn(Future future, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + try { + if (future.get() != null) { + return; + } else { + break; + } + } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn + try { + Thread.sleep(50); + } catch (InterruptedException e) { + i--; // Not a big deal to be interrupted, just go another time through the loop + } + } + fail("Should have received a non-null result from future without exception"); + } + + private void waitForUnmute(String topic, long waitMillis) { + for (int i = 0; i < waitMillis / 50; i++) { + if (!accumulator.isMutedTopic(topic)) { + return; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { } + } + fail(topic + " was never unmuted."); + } + + private PubsubSender newSender(String channelName, int retries) { + return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), + this.accumulator, + true, + MAX_REQUEST_SIZE, + retries, + metrics, + time, + REQUEST_TIMEOUT); + } + + private MockPubsubServer newServer(String channelName) { + MockPubsubServer out = new MockPubsubServer(); + try { + InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); + } catch (IOException e) { + return null; + } + return out; + } + +// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { +// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); +// Map partResp = Collections.singletonMap(tp, resp); +// return new ProduceResponse(partResp, throttleTimeMs); +// } + +} From 84a24c5f061f13e29880441d370ee1dccd38fc78 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Feb 2017 16:52:57 -0800 Subject: [PATCH 101/140] Add file PubsubConsumer --- .../clients/consumer/PubsubConsumer.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java new file mode 100644 index 00000000..0ab8947b --- /dev/null +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.google.kafka.clients.consumer; + +public class PubsubConsumer implements Consumer { + + private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; // this might need to be an /internal class + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + + +} \ No newline at end of file From 5fb9e8a639b0bb4f4fe340f774676930e518f294 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Feb 2017 14:36:16 -0800 Subject: [PATCH 102/140] Continuing to fill in consumer. --- .../clients/consumer/PubsubConsumer.java | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 0ab8947b..fb4faabd 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -25,6 +25,135 @@ public class PubsubConsumer implements Consumer { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + // currentThread holds the threadId of the current thread accessing PubsubConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Pubsub consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + } + } } \ No newline at end of file From 5bbe3a86eba2fa5d86fac5543a5d2368f0f54816 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Feb 2017 14:36:59 -0800 Subject: [PATCH 103/140] Switching branches, adding method stubs for consumer --- .../clients/consumer/PubsubConsumer.java | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index fb4faabd..42b3f7db 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -107,53 +107,6 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Pubsub consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - } } } \ No newline at end of file From 30bcd1f411a4d0fc1afc77445a002d440f1674ed Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 10:33:30 -0500 Subject: [PATCH 104/140] Added PubsubConsumer class --- .../clients/consumer/PubsubConsumer.java | 743 +++++++++++++++--- 1 file changed, 647 insertions(+), 96 deletions(-) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java index 42b3f7db..12bb1ec5 100644 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java @@ -10,103 +10,654 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.consumer; +package com.google.kafka.cients.consumer; public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactory.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; // this might need to be an /internal class - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - // currentThread holds the threadId of the current thread accessing PubsubConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + // this.valueDeserializer = config.getConfigured + } + } // left off at kafka's line 645 + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file From 8ea84c95d40837837319288618be2444f559b29e Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Feb 2017 14:21:45 -0800 Subject: [PATCH 105/140] New maven setup for mapped api --- .../clients/consumer/PubsubConsumer.java | 663 --------- .../clients/producer/PubsubProducer.java | 656 --------- .../clients/consumer/PubsubConsumer.java | 1277 ++++++++++------- .../clients/producer/PubsubProducer.java | 911 +++++++----- .../producer/internals/PubsubAccumulator.java | 0 .../producer/internals/PubsubBatch.java | 0 .../producer/internals/PubsubSender.java | 0 .../clients/producer/PubsubProducerTest.java | 0 .../producer/internals/MockPubsubServer.java | 0 .../internals/PubsubAccumulatorTest.java | 0 .../producer/internals/PubsubSenderTest.java | 0 11 files changed, 1322 insertions(+), 2185 deletions(-) delete mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java delete mode 100644 mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulator.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubBatch.java (100%) rename {mapped-api/clients/src/main/java/com/google/kafka => pubsub-mapped-api/src/main/java/com/google/pubsub}/clients/producer/internals/PubsubSender.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/PubsubProducerTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/MockPubsubServer.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubAccumulatorTest.java (100%) rename {mapped-api/clients/src/test/java/com/google/kafka => pubsub-mapped-api/src/test/java/com/google/pubsub}/clients/producer/internals/PubsubSenderTest.java (100%) diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java deleted file mode 100644 index 12bb1ec5..00000000 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +++ /dev/null @@ -1,663 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.cients.consumer; - -public class PubsubConsumer implements Consumer { - - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - // this.valueDeserializer = config.getConfigured - } - } // left off at kafka's line 645 - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } -} \ No newline at end of file diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java b/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java deleted file mode 100644 index 2077a3c1..00000000 --- a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/PubsubProducer.java +++ /dev/null @@ -1,656 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer; - -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; -import org.apache.kafka.common.utils.SystemTime; -import org.apache.kafka.common.utils.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.SocketOptions; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A Kafka client that publishes records to Google Cloud Pub/Sub. - */ -public class PubsubProducer implements Producer { - - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } - - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); - } - - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } - } - - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); - } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); - } - - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } - } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); - } - - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } - - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; - } - - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); - } - - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); - } - - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); - } - - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } - - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - } - - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } - - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 285c5d8e..cadfbf1b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,31 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ +<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java +package com.google.kafka.cients.consumer; +======= -package com.google.pubsub.clients.consumer; +package com.google.kafka.clients.consumer; -import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.OffsetAndTimestamp; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; +import org.apache.kafka.clients.consumer.internals.Fetcher; +import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; +import org.apache.kafka.clients.consumer.internals.SubscriptionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -34,6 +36,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -68,523 +71,723 @@ public class PubsubConsumer implements Consumer { - public PubsubConsumer(Map configs) { - - } - - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - public PubsubConsumer(Properties properties) { - - } - - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, - OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public OffsetAndMetadata committed(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the metrics kept by the consumer - */ - public Map metrics() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public List partitionsFor(String topic) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public Map> listTopics() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - public void pause(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - public void resume(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - public Set paused() { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - public Map offsetsForTimes(Map timestampsToSearch) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - public Map beginningOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - public Map endOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - throw new NotImplementedException("Not yet implemented"); - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - public void wakeup() { - throw new NotImplementedException("Not yet implemented"); - } + private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); + private static final long NO_CURRENT_THREAD = -1L; + private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "pubsub.consumer"; + static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; + + private final String clientId; + private final ConsumerCoordinator coordinator; + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Fetcher fetcher; + private final ConsumerInterceptors interceptors; + + private final Time time; + private final ConsumerNetworkClient client; + private final Metrics metrics; + private final SubscriptionState subscriptions; + private final Metadata metadata; // not sure if pubsub will have metadata + private final long retryBackoffMs; + private final long requestTimeoutMs; + private volatile boolean closed = false; + + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refcount = new AtomicInteger(0); + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + */ + public PubsubConsumer(Map configs) { + this(configs, null, null); + } + + /** + * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param configs The consumer configs + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + */ + public PubsubConsumer(Properties properties) { + this(properties, null, null); + } + + /** + * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a + * key and a value {@link Deserializer}. + *

+ * Valid configuration strings are documented at {@link ConsumerConfig} + * + * @param properties The consumer configuration properties + * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method + * won't be called in the consumer when the deserializer is passed in directly. + */ + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); + } + + @SuppressWarnings("unchecked") + private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + /* try { + log.debug("Starting the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) + throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + this.time = Time.SYSTEM; + + String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); + this.clientId = clientId; + Map metricsTags = new LinkedHashMap<>(); + metricsTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricsTags); + List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + // load interceptors and make sure they get clientId + Map userProvidedConfigs = config.originals(); + userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, + ConsumerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + if (keyDeserializer == null) { + this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.keyDeserializer.configure(config.originals(), true); + } else { + config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); + this.keyDeserializer = keyDeserializer; + } + if (valueDeserializer == null) { + this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + Deserializer.class); + this.valueDeserializer.configure(config.originals(), false); + } else { + config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); + this.valueDeserializer = valueDeserializer; + } + + ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); + this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); + this.metadata.update(Cluster.bootstrap(addresses), 0); + String metricGrpPrefix = "consumer"; + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + NetworkClient netClient = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), + this.metadata, + clientId, + 100, // a fixed large enough value will suffice + config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), + config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), + time, + true); + this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, + config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); + OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); + this.subscriptions = new SubscriptionState(offsetResetStrategy); + List assignors = config.getConfiguredInstances( + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, + PartitionAssignor.class); + this.coordinator = new ConsumerCoordinator(this.client, + config.getString(ConsumerConfig.GROUP_ID_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), + config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), + config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), + assignors, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + retryBackoffMs, + config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); + this.fetcher = new Fetcher<>(this.client, + config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), + config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), + config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), + config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), + config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), + this.keyDeserializer, + this.valueDeserializer, + this.metadata, + this.subscriptions, + metrics, + metricGrpPrefix, + this.time, + this.retryBackoffMs); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + + log.debug("Kafka consumer created"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed + // this is to prevent resource leak. see KAFKA-2121 + close(0, true); + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + + } */ + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + + } + + /** + * Get the metrics kept by the consumer + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public List partitionsFor(String topic) { + + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + @Override + public Map> listTopics() { + + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + @Override + public void pause(Collection partitions) { + + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + @Override + public void resume(Collection partitions) { + + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + @Override + public Set paused() { + + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + @Override + public Map beginningOffsets(Collection partitions) { + + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + @Override + public Map endOffsets(Collection partitions) { + + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + @Override + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + @Override + public void wakeup() { + + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index e9ca5753..5f7d3507 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,403 +1,656 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ - -package com.google.pubsub.clients.producer; - -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOException; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; -import org.apache.kafka.clients.producer.internals.ProduceRequestResult; -import org.apache.kafka.clients.producer.internals.RecordAccumulator; -import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +package com.google.kafka.clients.producer; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; +import com.google.kafka.clients.producer.internals.PubsubAccumulator; +import com.google.kafka.clients.producer.internals.PubsubSender; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.JmxReporter; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; +import java.net.SocketOptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - - private final PublisherFutureStub publisher; - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final int batchSize; - private final boolean isAcks; - private final Map> perTopicBatches; - private final int maxRequestSize; - private final Time time; - private final PubsubChannelUtil channelUtil; - - private boolean closed = false; - - private PubsubProducer(Builder builder) { - publisher = builder.publisher; - project = builder.project; - batchSize = builder.batchSize; - isAcks = builder.isAcks; - perTopicBatches = builder.perTopicBatches; - maxRequestSize = builder.maxRequestSize; - time = builder.time; - channelUtil = builder.channelUtil; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; - } - - public PubsubProducer(Map configs) { - this(new PubsubProducerConfig(configs), null, null); - } - - public PubsubProducer(Map configs, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - public PubsubProducer(Properties properties) { - this(new PubsubProducerConfig(properties), null, null); - } - - public PubsubProducer(Properties properties, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { - try { - log.trace("Starting the Pubsub producer"); - this.time = new SystemTime(); - channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); - - if (keySerializer == null) { - this.keySerializer = - configs.getConfiguredInstance( - PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.keySerializer.configure(configs.originals(), true); - } else { - configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - - if (valueSerializer == null) { - this.valueSerializer = configs.getConfiguredInstance( - PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.valueSerializer.configure(configs.originals(), false); - } else { - configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - } catch (Exception e) { - throw new RuntimeException(e); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); + private static final String JMX_PREFIX = "cps.producer"; + + private String clientId; + private final int maxRequestSize; + private final long totalMemorySize; + private final PubsubAccumulator accumulator; + private final PubsubSender sender; + private final Metrics metrics; + private final Thread ioThread; + private final CompressionType compressionType; + private final Sensor errors; + private final Time time; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerConfig producerConfig; + private final long maxBlockTimeMs; + private final int requestTimeoutMs; + private final ProducerInterceptors interceptors; + + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. Values can be + * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the + * string "42" or the integer 42). + * @param configs The producer configs + * + */ + public PubsubProducer(Map configs) { + this(new ProducerConfig(configs), null, null); } - batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); - isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); - project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - - log.debug("Producer successfully initialized."); - } - - /** - * Send the given record asynchronously and return a future which will eventually contain the response information. - * - * @param record The record to send - * @return A future which will eventually contain the response information - */ - public Future send(ProducerRecord record) { - return send(record, null); - } - - /** - * Send a record and invoke the given callback when the record has been acknowledged by the server - */ - public Future send(ProducerRecord record, Callback callback) { - log.trace("Received " + record.toString()); - if (closed) { - throw new RuntimeException("Publisher is closed"); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept + * either the string "42" or the integer 42). + * @param configs The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); } - String topic = record.topic(); - Map attributes = new HashMap<>(); - - byte[] serializedKey = ByteString.EMPTY.toByteArray(); - if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings + * are documented here. + * @param properties The producer configs + */ + public PubsubProducer(Properties properties) { + this(new ProducerConfig(properties), null, null); } - if (project == null) { - throw new RuntimeException("No project specified."); + /** + * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. + * Valid configuration strings are documented here. + * @param properties The producer configs + * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be + * called in the producer when the serializer is passed in directly. + * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't + * be called in the producer when the serializer is passed in directly. + */ + public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); } - byte[] valueBytes = ByteString.EMPTY.toByteArray(); - if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic, record.value()); - } + @SuppressWarnings({"unchecked", "deprecation"}) + private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { + log.trace("Starting the Kafka producer"); + Map userProvidedConfigs = config.originals(); + this.producerConfig = config; + this.time = new SystemTime(); + + clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (clientId.length() <= 0) + clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); + Map metricTags = new LinkedHashMap(); + metricTags.put("client-id", clientId); + MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) + .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) + .tags(metricTags); + List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, + MetricsReporter.class); + reporters.add(new JmxReporter(JMX_PREFIX)); + this.metrics = new Metrics(metricConfig, reporters, time); + long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + if (keySerializer == null) { + this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.keySerializer.configure(config.originals(), true); + } else { + config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + if (valueSerializer == null) { + this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + Serializer.class); + this.valueSerializer.configure(config.originals(), false); + } else { + config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } - checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); - - PubsubMessage message = - PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(valueBytes)) - .putAllAttributes(attributes) - .build(); - List batch = perTopicBatches.get(topic); - if (batch == null) { - batch = new ArrayList<>(batchSize); - perTopicBatches.put(topic, batch); - } - batch.add(message); - - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - RecordAccumulator.RecordAppendResult result = new RecordAppendResult( - new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); - if (result.batchIsFull) { - log.trace("Sending a batch of messages."); - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); - doSend(request, callback, result); - } - return result.future; - } - - private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { - try { - ListenableFuture response = publisher.publish(request); - if (callback != null) { - if (isAcks) { - Futures.addCallback( - response, - new FutureCallback() { - public void onSuccess(PublishResponse response) { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } + // load interceptors and make sure they get clientId + userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + ProducerInterceptor.class); + this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); + this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); + /* check for user defined settings. + * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. + * This should be removed with release 0.9 when the deprecated configs are removed. + */ + if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { + log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); + if (blockOnBufferFull) { + this.maxBlockTimeMs = Long.MAX_VALUE; + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } + } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + + "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); + this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); + } else { + this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); + } - public void onFailure(Throwable t) { - callback.onCompletion(null, new Exception(t)); - } - } - ); + /* check for user defined settings. + * If the TIME_OUT config is set use that for request timeout. + * This should be removed with release 0.9 + */ + if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { + log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); } else { - perTopicBatches.clear(); - callback.onCompletion(null, null); + this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); } - } else { - response.get(); - perTopicBatches.clear(); - } - } catch (InterruptedException | ExecutionException e) { - return new FutureFailure(e); + + this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), + this.totalMemorySize, + this.compressionType, + config.getLong(ProducerConfig.LINGER_MS_CONFIG), + retryBackoffMs, + metrics, + time); + + int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + sendBufferSize = SocketOptions.SO_SNDBUF; + int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); + if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) + receiveBufferSize = SocketOptions.SO_RCVBUF; + + ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) + .flowControlWindow(sendBufferSize + receiveBufferSize) + .executor(new ThreadPoolExecutor(1, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); + this.sender = new PubsubSender(channel, + this.accumulator, + config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, + config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), + config.getInt(ProducerConfig.RETRIES_CONFIG), + this.metrics, + new SystemTime(), + this.requestTimeoutMs); + String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); + this.ioThread = new KafkaThread(ioThreadName, this.sender, true); + this.ioThread.start(); + + this.errors = this.metrics.sensor("errors"); + + config.logUnused(); + AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); + log.debug("Pubsub producer started"); } - return result.future; - } - private void checkRecordSize(int size) { - if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" - + " configured"); + private static int parseAcks(String acksString) { + try { + return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid configuration value for 'acks': " + acksString); + } } - } - - /** - * Flush any accumulated records from the producer. Blocks until all sends are complete. - */ - public void flush() { - log.debug("Flushing..."); - for (String topic : perTopicBatches.keySet()) { - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); - doSend(request, null, null); + + /** + * Asynchronously send a record to a topic. Equivalent to send(record, null). + * See {@link #send(ProducerRecord, Callback)} for details. + */ + @Override + public Future send(ProducerRecord record) { + return send(record, null); } - } - - /** - * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change - * over time so this list should not be cached. - */ - public List partitionsFor(String topic) { - throw new NotImplementedException("Partitions not supported"); - } - - /** - * Return a map of metrics maintained by the producer - */ - public Map metrics() { - throw new NotImplementedException("Metrics not supported."); - } - - /** - * Close this producer - */ - public void close() { - close(0, null); - } - - /** - * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the - * timeout, fail any pending send requests and force close the producer. - */ - public void close(long timeout, TimeUnit unit) { - if (timeout < 0) { - throw new IllegalArgumentException("Timeout cannot be negative."); + + /** + * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. + *

+ * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of + * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the + * response after each one. + *

+ * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset + * it was assigned and the timestamp of the record. If + * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp + * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the + * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the + * topic, the timestamp will be the Kafka broker local time when the message is appended. + *

+ * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the + * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() + * get()} on this future will block until the associated request completes and then return the metadata for the record + * or throw any exception that occurred while sending the record. + *

+ * If you want to simulate a simple blocking call you can call the get() method immediately: + * + * 

+     * {@code
+     * byte[] key = "key".getBytes();
+     * byte[] value = "value".getBytes();
+     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
+     * producer.send(record).get();
+     * }
+ *

+ * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that + * will be invoked when the request is complete. + * + * 

+     * {@code
+     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
+     * producer.send(myRecord,
+     *               new Callback() {
+     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
+     *                       if(e != null) {
+     *                          e.printStackTrace();
+     *                       } else {
+     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
+     *                       }
+     *                   }
+     *               });
+     * }
+     *
+ * + * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the + * following example callback1 is guaranteed to execute before callback2: + * + * 
+     * {@code
+     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
+     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
+     * }
+     *
+ *

+ * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or + * they will delay the sending of messages from other threads. If you want to execute blocking or computationally + * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body + * to parallelize processing. + * + * @param record The record to send + * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null + * indicates no callback) + * + * @throws InterruptException If the thread is interrupted while blocked + * @throws SerializationException If the key or value are not valid objects given the configured serializers + * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. + * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. + * + */ + @Override + public Future send(ProducerRecord record, Callback callback) { + // intercept the record, which can be potentially modified; this method does not throw exceptions + ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); + return doSend(interceptedRecord, callback); } - channelUtil.closeChannel(); - log.debug("Closed producer"); - closed = true; - } + /** + * Implementation of asynchronously send a record to a topic. + */ + private Future doSend(ProducerRecord record, Callback callback) { + TopicPartition tp = null; + try { + byte[] serializedKey; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + byte[] serializedValue; + try { + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + + int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); + ensureValidRecordSize(serializedSize); + tp = new TopicPartition(record.topic(), 0); + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); + // producer callback will make sure to call both 'callback' and interceptor callback + Callback interceptCallback = + this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, + serializedValue, interceptCallback, maxBlockTimeMs); + return result.future; + // handling exceptions and record the errors; + // for API exceptions return them in the future, + // for other exceptions throw directly + } catch (ApiException e) { + log.debug("Exception occurred during message send:", e); + if (callback != null) + callback.onCompletion(null, e); + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + return new FutureFailure(e); + } catch (InterruptedException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw new InterruptException(e); + } catch (BufferExhaustedException e) { + this.errors.record(); + this.metrics.sensor("buffer-exhausted-records").record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (KafkaException e) { + this.errors.record(); + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } catch (Exception e) { + // we notify interceptor about all exceptions, since onSend is called before anything else in this method + if (this.interceptors != null) + this.interceptors.onSendError(record, tp, e); + throw e; + } + } - public static class Builder { - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + /** + * Validate that the record size isn't too large + */ + private void ensureValidRecordSize(int size) { + if (size > this.maxRequestSize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the maximum request size you have configured with the " + + ProducerConfig.MAX_REQUEST_SIZE_CONFIG + + " configuration."); + if (size > this.totalMemorySize) + throw new RecordTooLargeException("The message is " + size + + " bytes when serialized which is larger than the total memory buffer you have configured with the " + + ProducerConfig.BUFFER_MEMORY_CONFIG + + " configuration."); + } - private PubsubChannelUtil channelUtil; - private PublisherFutureStub publisher; - private int batchSize; - private boolean isAcks; - private Map> perTopicBatches; - private int maxRequestSize; - private Time time; - - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { - Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); - this.project = project; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - setDefaults(); + /** + * Invoking this method makes all buffered records immediately available to send (even if linger.ms is + * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition + * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). + * A request is considered completed when it is successfully acknowledged + * according to the acks configuration you have specified or else it results in an error. + *

+ * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, + * however no guarantee is made about the completion of records sent after the flush call begins. + *

+ * This method can be useful when consuming from some input system and producing into Kafka. The flush() call + * gives a convenient way to ensure all previously sent messages have actually completed. + *

+ * This example shows how to consume from one Kafka topic and produce to another Kafka topic: + * 

+     * {@code
+     * for(ConsumerRecord record: consumer.poll(100))
+     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
+     * producer.flush();
+     * consumer.commit();
+     * }
+     *
+ * + * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur + * we need to set retries=<large_number> in our config. + * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void flush() { + log.trace("Flushing accumulated records in producer."); + this.accumulator.beginFlush(); + try { + this.accumulator.awaitFlushCompletion(); + } catch (InterruptedException e) { + throw new InterruptException("Flush interrupted.", e); + } + } + + /** + * Get the partition metadata for the give topic. This can be used for custom partitioning. + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public List partitionsFor(String topic) { + List out = new ArrayList<>(1); + out.add(new PartitionInfo(topic, 0, null, null, null)); + return out; } - private void setDefaults() { - // this is where to set 'regular' fields w/o side effects - this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; - this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; - this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; - this.time = new SystemTime(); + /** + * Get the full set of internal metrics maintained by the producer. + */ + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); } - public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + /** + * Close this producer. This method blocks until all previously sent requests complete. + * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). + *

+ * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) + * will be called instead. We do this because the sender thread would otherwise try to join itself and + * block forever. + *

+ * + * @throws InterruptException If the thread is interrupted while blocked + */ + @Override + public void close() { + close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } - public Builder batchSize(int val) { - Preconditions.checkArgument(val > 0); - batchSize = val; - return this; + /** + * This method waits up to timeout for the producer to complete the sending of all incomplete requests. + *

+ * If the producer is unable to complete all requests before the timeout expires, this method will fail + * any unsent and unacknowledged records immediately. + *

+ * If invoked from within a {@link Callback} this method will not block and will be equivalent to + * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while + * blocking the I/O thread of the producer. + * + * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be + * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted while blocked + * @throws IllegalArgumentException If the timeout is negative. + */ + @Override + public void close(long timeout, TimeUnit timeUnit) { + close(timeout, timeUnit, false); } - public Builder isAcks(boolean val) { isAcks = val; return this; } + private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { + if (timeout < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); + // this will keep track of the first encountered exception + AtomicReference firstException = new AtomicReference(); + boolean invokedFromCallback = Thread.currentThread() == this.ioThread; + if (timeout > 0) { + if (invokedFromCallback) { + log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + + "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); + } else { + // Try to close gracefully. + if (this.sender != null) + this.sender.initiateClose(); + if (this.ioThread != null) { + try { + this.ioThread.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException t) { + firstException.compareAndSet(null, t); + log.error("Interrupted while joining ioThread", t); + } + } + } + } - public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { + log.info("Proceeding to force close the producer since pending requests could not be completed " + + "within timeout {} ms.", timeout); + this.sender.forceClose(); + // Only join the sender thread when not calling from callback. + if (!invokedFromCallback) { + try { + this.ioThread.join(); + } catch (InterruptedException e) { + firstException.compareAndSet(null, e); + } + } + } - public Builder maxRequestSize(int val) { - Preconditions.checkArgument(val >= 0); - maxRequestSize = val; - return this; + ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); + ClientUtils.closeQuietly(metrics, "producer metrics", firstException); + ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); + ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); + AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); + log.debug("The Kafka producer has closed."); + if (firstException.get() != null && !swallowException) + throw new KafkaException("Failed to close kafka producer", firstException.get()); } - public Builder time(Time val) { time = val; return this; } + private static class FutureFailure implements Future { - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + private final ExecutionException exception; - public PubsubProducer build() { - // this is where to set fields w/ side effects - if (channelUtil == null) { - this.channelUtil = new PubsubChannelUtil(); - } - if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); - } - return new PubsubProducer(this); - } - } + public FutureFailure(Exception exception) { + this.exception = new ExecutionException(exception); + } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ - private static class FutureFailure implements Future { - private final ExecutionException exception; + @Override + public boolean cancel(boolean interrupt) { + return false; + } - public FutureFailure(Exception e) { - this.exception = new ExecutionException(e); - } + @Override + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } - public boolean cancel(boolean interrupt) { - return false; - } + @Override + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; + } - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + @Override + public boolean isCancelled() { + return false; + } - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + @Override + public boolean isDone() { + return true; + } - public boolean isCancelled() { - return false; } - public boolean isDone() { - return true; + /** + * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and + * notifies producer interceptors about the request completion. + */ + private static class InterceptorCallback implements Callback { + private final Callback userCallback; + private final ProducerInterceptors interceptors; + private final TopicPartition tp; + + public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, + TopicPartition tp) { + this.userCallback = userCallback; + this.interceptors = interceptors; + this.tp = tp; + } + + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (this.interceptors != null) { + if (metadata == null) { + this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), + exception); + } else { + this.interceptors.onAcknowledgement(metadata, exception); + } + } + if (this.userCallback != null) + this.userCallback.onCompletion(metadata, exception); + } } - } } diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubAccumulator.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubBatch.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java diff --git a/mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java similarity index 100% rename from mapped-api/clients/src/main/java/com/google/kafka/clients/producer/internals/PubsubSender.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/PubsubProducerTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/MockPubsubServer.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubAccumulatorTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java diff --git a/mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java similarity index 100% rename from mapped-api/clients/src/test/java/com/google/kafka/clients/producer/internals/PubsubSenderTest.java rename to pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java From 38d3ebae01d5db3ac1e1411fa817b0c7724b3b65 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 15 Feb 2017 06:54:12 -0800 Subject: [PATCH 106/140] scratching some of the mapped api to make it more pub/sub --- .../clients/consumer/PubsubConsumer.java | 119 +--- .../clients/producer/PubsubProducer.java | 36 +- .../producer/internals/PubsubAccumulator.java | 546 ------------------ .../producer/internals/PubsubBatch.java | 181 ------ .../producer/internals/PubsubSender.java | 524 ----------------- 5 files changed, 4 insertions(+), 1402 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index cadfbf1b..72eb4413 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -14,7 +14,7 @@ package com.google.kafka.cients.consumer; ======= -package com.google.kafka.clients.consumer; +package com.google.pubsub.clients.consumer; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; @@ -162,121 +162,7 @@ public PubsubConsumer(Properties properties, @SuppressWarnings("unchecked") private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - /* try { - log.debug("Starting the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); - int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - if (this.requestTimeoutMs <= sessionTimeoutMs || this.requestTimeoutMs <= fetchMaxWaitMs) - throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); - this.time = Time.SYSTEM; - - String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); - this.clientId = clientId; - Map metricsTags = new LinkedHashMap<>(); - metricsTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricsTags); - List reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - - // load interceptors and make sure they get clientId - Map userProvidedConfigs = config.originals(); - userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, - ConsumerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); - if (keyDeserializer == null) { - this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.keyDeserializer.configure(config.originals(), true); - } else { - config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); - this.keyDeserializer = keyDeserializer; - } - if (valueDeserializer == null) { - this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - Deserializer.class); - this.valueDeserializer.configure(config.originals(), false); - } else { - config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); - this.valueDeserializer = valueDeserializer; - } - - ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); - this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - this.metadata.update(Cluster.bootstrap(addresses), 0); - String metricGrpPrefix = "consumer"; - ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - NetworkClient netClient = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), - this.metadata, - clientId, - 100, // a fixed large enough value will suffice - config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), - config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), - config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), - time, - true); - this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, - config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); - OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); - this.subscriptions = new SubscriptionState(offsetResetStrategy); - List assignors = config.getConfiguredInstances( - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - PartitionAssignor.class); - this.coordinator = new ConsumerCoordinator(this.client, - config.getString(ConsumerConfig.GROUP_ID_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), - config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), - config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), - assignors, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - retryBackoffMs, - config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); - this.fetcher = new Fetcher<>(this.client, - config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), - config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), - config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), - config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), - config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), - this.keyDeserializer, - this.valueDeserializer, - this.metadata, - this.subscriptions, - metrics, - metricGrpPrefix, - this.time, - this.retryBackoffMs); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - - log.debug("Kafka consumer created"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed - // this is to prevent resource leak. see KAFKA-2121 - close(0, true); - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - - } */ + } /** @@ -329,7 +215,6 @@ public Set subscription() { * subscribed topics * @throws IllegalArgumentException If topics is null or contains null or empty elements */ - @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 5f7d3507..c9c26b63 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -10,11 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.ProducerConfig; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import com.google.kafka.clients.producer.internals.PubsubAccumulator; import com.google.kafka.clients.producer.internals.PubsubSender; @@ -87,52 +88,19 @@ public class PubsubProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - * @param configs The producer configs - * - */ public PubsubProducer(Map configs) { this(new ProducerConfig(configs), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept - * either the string "42" or the integer 42). - * @param configs The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. - * @param properties The producer configs - */ public PubsubProducer(Properties properties) { this(new ProducerConfig(properties), null, null); } - /** - * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. - * Valid configuration strings are documented here. - * @param properties The producer configs - * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be - * called in the producer when the serializer is passed in directly. - * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't - * be called in the producer when the serializer is passed in directly. - */ public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java deleted file mode 100644 index e8ff1bba..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubAccumulator.java +++ /dev/null @@ -1,546 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.CopyOnWriteMap; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class acts as a queue that accumulates records into {@link com.google.pubsub.v1.PubsubMessage} - * instances to be sent to the server. - *

- * The accumulator uses a bounded amount of memory and append calls will block when that memory is exhausted, unless - * this behavior is explicitly disabled. - */ -public final class PubsubAccumulator { - private static final Logger log = LoggerFactory.getLogger(RecordAccumulator.class); - - private int mutedcalls = 0; - private volatile boolean closed; - private final AtomicInteger flushesInProgress; - private final AtomicInteger appendsInProgress; - private final int batchSize; - private final CompressionType compression; - private final long lingerMs; - private final long retryBackoffMs; - private final BufferPool free; - private final Time time; - private final ConcurrentMap> batches; - private final PubsubAccumulator.IncompleteBatches incomplete; - // The following variables are only accessed by the sender thread, so we don't need to protect them. - private final Set muted; - - /** - * Create a new record accumulator - * - * @param batchSize The size to use when allocating {@link com.google.pubsub.v1.PubsubMessage} instances - * @param totalSize The maximum memory the record accumulator can use. - * @param compression The compression codec for the records - * @param lingerMs An artificial delay time to add before declaring a records instance that isn't full ready for - * sending. This allows time for more records to arrive. Setting a non-zero lingerMs will trade off some - * latency for potentially better throughput due to more batching (and hence fewer, larger requests). - * @param retryBackoffMs An artificial delay time to retry the produce request upon receiving an error. This avoids - * exhausting all retries in a short period of time. - * @param metrics The metrics - * @param time The time instance to use - */ - public PubsubAccumulator(int batchSize, - long totalSize, - CompressionType compression, - long lingerMs, - long retryBackoffMs, - Metrics metrics, - Time time) { - this.closed = false; - this.flushesInProgress = new AtomicInteger(0); - this.appendsInProgress = new AtomicInteger(0); - this.batchSize = batchSize; - this.compression = compression; - this.lingerMs = lingerMs; - this.retryBackoffMs = retryBackoffMs; - this.batches = new CopyOnWriteMap<>(); - String metricGrpName = "producer-metrics"; - this.free = new BufferPool(totalSize, batchSize, metrics, time, metricGrpName); - this.incomplete = new PubsubAccumulator.IncompleteBatches(); - this.muted = new HashSet<>(); - this.time = time; - registerMetrics(metrics, metricGrpName); - } - - private void registerMetrics(Metrics metrics, String metricGrpName) { - MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); - Measurable waitingThreads = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.queued(); - } - }; - metrics.addMetric(metricName, waitingThreads); - - metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); - Measurable totalBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.totalMemory(); - } - }; - metrics.addMetric(metricName, totalBytes); - - metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); - Measurable availableBytes = new Measurable() { - public double measure(MetricConfig config, long now) { - return free.availableMemory(); - } - }; - metrics.addMetric(metricName, availableBytes); - - Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); - metricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); - bufferExhaustedRecordSensor.add(metricName, new Rate()); - } - - /** - * Add a record to the accumulator, return the append result - *

- * The append result will contain the future metadata, and flag for whether the appended batch is full or a new batch is created - *

- * - * @param timestamp The timestamp of the record - * @param key The key for the record - * @param value The value for the record - * @param callback The user-supplied callback to execute when the request is complete - * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available - */ - public PubsubAccumulator.RecordAppendResult append(String topic, - long timestamp, - byte[] key, - byte[] value, - Callback callback, - long maxTimeToBlock) throws InterruptedException { - // We keep track of the number of appending thread to make sure we do not miss batches in - // abortIncompleteBatches(). - appendsInProgress.incrementAndGet(); - try { - Deque deque = getOrCreateDeque(topic); - synchronized (deque) { - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - return appendResult; - } - } - - // we don't have an in-progress record batch try to allocate a new batch - int size = Math.max(this.batchSize, Records.LOG_OVERHEAD + Record.recordSize(key, value)); - log.trace("Allocating a new {} byte message buffer for topic {}", size, topic); - ByteBuffer buffer = free.allocate(size, maxTimeToBlock); - synchronized (deque) { - // Need to check if producer is closed again after grabbing the dequeue lock. - if (closed) { - throw new IllegalStateException("Cannot send after the producer is closed."); - } - - PubsubAccumulator.RecordAppendResult appendResult = tryAppend(timestamp, key, value, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... - free.deallocate(buffer); - return appendResult; - } - MemoryRecords records = MemoryRecords.emptyRecords(buffer, compression, this.batchSize); - PubsubBatch batch = new PubsubBatch(topic, records, time.milliseconds()); - FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, callback, time.milliseconds())); - - deque.addLast(batch); - incomplete.add(batch); - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || batch.records.isFull(), true); - } - } finally { - appendsInProgress.decrementAndGet(); - } - } - - /** - * If `RecordBatch.tryAppend` fails (i.e. the record batch is full), close its memory records to release temporary - * resources (like compression streams buffers). - */ - private PubsubAccumulator.RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, - Deque deque) { - PubsubBatch last = deque.peekLast(); - if (last != null) { - FutureRecordMetadata future = last.tryAppend(timestamp, key, value, callback, time.milliseconds()); - if (future == null) - last.records.close(); - else - return new PubsubAccumulator.RecordAppendResult(future, deque.size() > 1 || last.records.isFull(), false); - } - return null; - } - - /** - * Abort the batches that have been sitting in RecordAccumulator for more than the configured requestTimeout - * due to metadata being unavailable - */ - public List abortExpiredBatches(int requestTimeout, long now) { - List expiredBatches = new ArrayList<>(); - int count = 0; - for (Map.Entry> entry : this.batches.entrySet()) { - Deque dq = entry.getValue(); - String topic = entry.getKey(); - // We only check if the batch should be expired if the partition does not have a batch in flight. - // This is to prevent later batches from being expired while an earlier batch is still in progress. - // Note that `muted` is only ever populated if `max.in.flight.request.per.connection=1` so this protection - // is only active in this case. Otherwise the expiration order is not guaranteed. - if (!muted.contains(topic)) { - synchronized (dq) { - // iterate over the batches and expire them if they have been in the accumulator for more than requestTimeOut - PubsubBatch lastBatch = dq.peekLast(); - Iterator batchIterator = dq.iterator(); - while (batchIterator.hasNext()) { - PubsubBatch batch = batchIterator.next(); - boolean isFull = batch != lastBatch || batch.records.isFull(); - // check if the batch is expired - if (batch.maybeExpire(requestTimeout, retryBackoffMs, now, this.lingerMs, isFull)) { - expiredBatches.add(batch); - count++; - batchIterator.remove(); - deallocate(batch); - } else { - // Stop at the first batch that has not expired. - break; - } - } - } - } - } - if (!expiredBatches.isEmpty()) - log.trace("Expired {} batches in accumulator", count); - - return expiredBatches; - } - - /** - * Re-enqueue the given record batch in the accumulator to retry - */ - public void reenqueue(PubsubBatch batch, long now) { - batch.attempts++; - batch.lastAttemptMs = now; - batch.lastAppendTime = now; - batch.setRetry(); - Deque deque = getOrCreateDeque(batch.topic); - synchronized (deque) { - deque.addFirst(batch); - } - } - - /** - * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable - * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated - * partition batches. - *

- * A destination node is ready to send data if: - *

    - *
  1. There is at least one partition that is not backing off its send - *
  2. and those partitions are not muted (to prevent reordering if - * {@value org.apache.kafka.clients.producer.ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION} - * is set to one)
    3. - *
  3. and any of the following are true
    4. - *
      - *
    • The record set is full
      • - *
    • The record set has sat in the accumulator for at least lingerMs milliseconds
      • - *
    • The accumulator is out of memory and threads are blocking waiting for data (in this case all partitions - * are immediately considered ready).
      • - *
    • The accumulator has been closed
      • - *
    - *
- */ - public Set ready(long nowMs) { - Set readyTopics = new HashSet<>(); - - boolean exhausted = this.free.queued() > 0; - for (Map.Entry> entry : this.batches.entrySet()) { - String topic = entry.getKey(); - Deque deque = entry.getValue(); - - synchronized (deque) { - if (!muted.contains(topic)) { - PubsubBatch batch = deque.peekFirst(); - if (batch != null) { - boolean backingOff = batch.attempts > 0 && batch.lastAttemptMs + retryBackoffMs > nowMs; - long waitedTimeMs = nowMs - batch.lastAttemptMs; - long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs; - long timeLeftMs = Math.max(timeToWaitMs - waitedTimeMs, 0); - boolean full = deque.size() > 1 || batch.records.isFull(); - boolean expired = waitedTimeMs >= timeToWaitMs; - boolean sendable = full || expired || exhausted || closed || flushInProgress(); - if (sendable && !backingOff) { - readyTopics.add(topic); - } - } - } - } - } - - return readyTopics; - } - - /** - * @return Whether there is any unsent record in the accumulator. - */ - public boolean hasUnsent() { - for (Map.Entry> entry : this.batches.entrySet()) { - Deque deque = entry.getValue(); - synchronized (deque) { - if (!deque.isEmpty()) - return true; - } - } - return false; - } - - /** - * Drain all the data and collates it into a list of batches that will fit within the specified size. - * - * @param maxSize The maximum number of bytes to drain - * @param now The current unix time in milliseconds - * @return A list of {@link PubsubBatch} with total size less than the requested maxSize. - */ - public Map> drain(Set topics, int maxSize, long now) { - if (topics.isEmpty()) { - return Collections.emptyMap(); - } - Map> out = new HashMap<>(); - for (String topic : topics) { - int size = 0; - List ready = new ArrayList<>(); - out.put(topic, ready); - if (muted.contains(topic)) { - continue; - } - Deque deque = getDeque(topic); - synchronized (deque) { - PubsubBatch first = deque.peekFirst(); - if (first != null) { - boolean backoff = first.attempts > 0 && first.lastAttemptMs + retryBackoffMs > now; - // Only drain the batch if it is not during backoff period. - if (!backoff) { - if (size + first.records.sizeInBytes() <= maxSize || out.isEmpty()) { - PubsubBatch batch = deque.pollFirst(); - batch.records.close(); - size += batch.records.sizeInBytes(); - ready.add(batch); - batch.drainedMs = now; - } - } - } - } - } - return out; - } - - private Deque getDeque(String topic) { - return batches.get(topic); - } - - /** - * Get the deque for the given topic-partition, creating it if necessary. - */ - private Deque getOrCreateDeque(String topic) { - Deque d = this.batches.get(topic); - if (d != null) - return d; - d = new ArrayDeque<>(); - Deque previous = this.batches.putIfAbsent(topic, d); - if (previous == null) - return d; - else - return previous; - } - - /** - * Deallocate the record batch - */ - public void deallocate(PubsubBatch batch) { - incomplete.remove(batch); - free.deallocate(batch.records.buffer(), batch.records.initialCapacity()); - } - - /** - * Are there any threads currently waiting on a flush? - * - * package private for test - */ - boolean flushInProgress() { - return flushesInProgress.get() > 0; - } - - /* Visible for testing */ - Map> batches() { - return Collections.unmodifiableMap(batches); - } - - /** - * Initiate the flushing of data from the accumulator...this makes all requests immediately ready - */ - public void beginFlush() { - this.flushesInProgress.getAndIncrement(); - } - - /** - * Are there any threads currently appending messages? - */ - private boolean appendsInProgress() { - return appendsInProgress.get() > 0; - } - - /** - * Mark all partitions as ready to send and block until the send is complete - */ - public void awaitFlushCompletion() throws InterruptedException { - try { - for (PubsubBatch batch : this.incomplete.all()) - batch.produceFuture.await(); - } finally { - this.flushesInProgress.decrementAndGet(); - } - } - - /** - * This function is only called when sender is closed forcefully. It will fail all the - * incomplete batches and return. - */ - public void abortIncompleteBatches() { - // We need to keep aborting the incomplete batch until no thread is trying to append to - // 1. Avoid losing batches. - // 2. Free up memory in case appending threads are blocked on buffer full. - // This is a tight loop but should be able to get through very quickly. - do { - abortBatches(); - } while (appendsInProgress()); - // After this point, no thread will append any messages because they will see the close - // flag set. We need to do the last abort after no thread was appending in case there was a new - // batch appended by the last appending thread. - abortBatches(); - this.batches.clear(); - } - - /** - * Go through incomplete batches and abort them. - */ - private void abortBatches() { - for (PubsubBatch batch : incomplete.all()) { - Deque deque = getDeque(batch.topic); - // Close the batch before aborting - synchronized (deque) { - batch.records.close(); - deque.remove(batch); - } - batch.done(-1L, Record.NO_TIMESTAMP, new IllegalStateException("Producer is closed forcefully.")); - deallocate(batch); - } - } - - public void muteTopic(String topic) { - mutedcalls++; - muted.add(topic); - } - - public void unmuteTopic(String topic) { - mutedcalls++; - muted.remove(topic); - } - - public boolean isMutedTopic(String topic) { - return muted.contains(topic); - } - - /** - * Close this accumulator and force all the record buffers to be drained - */ - public void close() { - this.closed = true; - } - - /* - * Metadata about a record just appended to the record accumulator - */ - public final static class RecordAppendResult { - public final FutureRecordMetadata future; - public final boolean batchIsFull; - public final boolean newBatchCreated; - - public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) { - this.future = future; - this.batchIsFull = batchIsFull; - this.newBatchCreated = newBatchCreated; - } - } - - /* - * A threadsafe helper class to hold RecordBatches that haven't been ack'd yet - */ - private final static class IncompleteBatches { - private final Set incomplete; - - public IncompleteBatches() { - this.incomplete = new HashSet(); - } - - public void add(PubsubBatch batch) { - synchronized (incomplete) { - this.incomplete.add(batch); - } - } - - public void remove(PubsubBatch batch) { - synchronized (incomplete) { - boolean removed = this.incomplete.remove(batch); - if (!removed) - throw new IllegalStateException("Remove from the incomplete set failed. This should be impossible."); - } - } - - public Iterable all() { - synchronized (incomplete) { - return new ArrayList<>(this.incomplete); - } - } - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java deleted file mode 100644 index 6f60d8fd..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubBatch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.record.MemoryRecords; -import org.apache.kafka.common.record.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A batch of records that is or will be sent. - * - * This class is not thread safe and external synchronization must be used when modifying it - */ -public final class PubsubBatch { - - private static final Logger log = LoggerFactory.getLogger(RecordBatch.class); - - public int recordCount = 0; - public int maxRecordSize = 0; - public volatile int attempts = 0; - public final long createdMs; - public long drainedMs; - public long lastAttemptMs; - public final MemoryRecords records; - public final ProduceRequestResult produceFuture; - public long lastAppendTime; - public String topic; - - private final List thunks; - private long offsetCounter = 0L; - private boolean retry; - - public PubsubBatch(String topic, MemoryRecords records, long now) { - this.createdMs = now; - this.lastAttemptMs = now; - this.records = records; - this.produceFuture = new ProduceRequestResult(); - this.thunks = new ArrayList(); - this.lastAppendTime = createdMs; - this.retry = false; - this.topic = topic; - } - - /** - * Append the record to the current record set and return the relative offset within that record set - * - * @return The RecordSend corresponding to this record or null if there isn't sufficient room. - */ - public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) { - if (!this.records.hasRoomFor(key, value)) { - return null; - } else { - long checksum = this.records.append(offsetCounter++, timestamp, key, value); - this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value)); - this.lastAppendTime = now; - FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount, - timestamp, checksum, - key == null ? -1 : key.length, - value == null ? -1 : value.length); - if (callback != null) - thunks.add(new Thunk(callback, future)); - this.recordCount++; - return future; - } - } - - /** - * Complete the request - * - * @param baseOffset The base offset of the messages assigned by the server - * @param timestamp The timestamp returned by the broker. - * @param exception The exception that occurred (or null if the request was successful) - */ - public void done(long baseOffset, long timestamp, RuntimeException exception) { - TopicPartition tp = new TopicPartition(topic, 0); - log.trace("Produced messages with base offset offset {} and error: {}.", - baseOffset, - exception); - // execute callbacks - for (int i = 0; i < this.thunks.size(); i++) { - try { - Thunk thunk = this.thunks.get(i); - if (exception == null) { - // If the timestamp returned by server is NoTimestamp, that means CreateTime is used. Otherwise LogAppendTime is used. - RecordMetadata metadata = new RecordMetadata(tp, baseOffset, thunk.future.relativeOffset(), - timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, - thunk.future.checksum(), - thunk.future.serializedKeySize(), - thunk.future.serializedValueSize()); - thunk.callback.onCompletion(metadata, null); - } else { - thunk.callback.onCompletion(null, exception); - } - } catch (Exception e) { - log.error("Error executing user-provided callback on message for topic {}:", topic, e); - } - } - this.produceFuture.done(tp, baseOffset, exception); - } - - /** - * A callback and the associated FutureRecordMetadata argument to pass to it. - */ - final private static class Thunk { - final Callback callback; - final FutureRecordMetadata future; - - public Thunk(Callback callback, FutureRecordMetadata future) { - this.callback = callback; - this.future = future; - } - } - - @Override - public String toString() { - return "RecordBatch(recordCount=" + recordCount + ")"; - } - - /** - * A batch whose metadata is not available should be expired if one of the following is true: - *
    - *
  1. the batch is not in retry AND request timeout has elapsed after it is ready (full or linger.ms has reached). - *
  2. the batch is in retry AND request timeout has elapsed after the backoff period ended. - *
- */ - public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) { - boolean expire = false; - String errorMessage = null; - - if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) { - expire = true; - errorMessage = (now - this.lastAppendTime) + " ms has passed since last append"; - } else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) { - expire = true; - errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time"; - } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) { - expire = true; - errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time"; - } - - if (expire) { - this.records.close(); - this.done(-1L, Record.NO_TIMESTAMP, - new TimeoutException("Expiring " + recordCount + " record(s) due to " + errorMessage)); - } - - return expire; - } - - /** - * Returns if the batch is been retried for sending to kafka - */ - public boolean inRetry() { - return this.retry; - } - - /** - * Set retry to true if the batch is being retried (for send) - */ - public void setRetry() { - this.retry = true; - } -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java deleted file mode 100644 index aaf0580e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubSender.java +++ /dev/null @@ -1,524 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.errors.RetriableException; -import org.apache.kafka.common.errors.TopicAuthorizationException; -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * The background thread that handles the sending of produce requests to Cloud Pub/Sub. This thread makes metadata - * requests to renew its view of the cluster and then sends produce requests to the appropriate nodes. - */ -public class PubsubSender implements Runnable { - private static final Logger log = LoggerFactory.getLogger(Sender.class); - - /* the record accumulator that batches records */ - private final PubsubAccumulator accumulator; - - /* the grpc stub to send records to pubsub */ - private final PublisherGrpc.PublisherFutureStub stub; - - private final ThreadPoolExecutor executor; - - /* the flag indicating whether the producer should guarantee the message order on the broker or not. */ - private final boolean guaranteeMessageOrder; - - /* the maximum request size to attempt to send to the server */ - private final int maxRequestSize; - - /* the number of times to retry a failed request before giving up */ - private final int retries; - - /* the clock instance used for getting the time */ - private final Time time; - - /* true while the sender thread is still running */ - private volatile boolean running; - - /* true when the caller wants to ignore all unsent/inflight messages and force close. */ - private volatile boolean forceClose; - - /* metrics */ - private final PubsubSenderMetrics sensors; - - /* the max time to wait for the server to respond to the request*/ - private final int requestTimeout; - - public PubsubSender(ManagedChannel channel, - PubsubAccumulator accumulator, - boolean guaranteeMessageOrder, - int maxRequestSize, - int retries, - Metrics metrics, - Time time, - int requestTimeout) { - this.accumulator = accumulator; - this.guaranteeMessageOrder = guaranteeMessageOrder; - this.maxRequestSize = maxRequestSize; - this.running = true; - this.retries = retries; - this.time = time; - this.sensors = new PubsubSenderMetrics(metrics); - this.requestTimeout = requestTimeout; - this.stub = PublisherGrpc.newFutureStub(channel) - .withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue()); - } - - @Override - public void run() { - log.debug("Starting Kafka producer I/O thread."); - - // main loop, runs until close is called - while (running) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - - log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); - - // okay we stopped accepting requests but there may still be - // requests in the accumulator or waiting for acknowledgment, - // wait until these are completed. - while (!forceClose && (this.accumulator.hasUnsent() || this.executor.getActiveCount() > 0)) { - try { - run(time.milliseconds()); - } catch (Exception e) { - log.error("Uncaught error in kafka producer I/O thread: ", e); - } - } - if (forceClose) { - // We need to fail all the incomplete batches and wake up the threads waiting on - // the futures. - this.accumulator.abortIncompleteBatches(); - } - try { - this.executor.shutdown(); - } catch (Exception e) { - log.error("Failed to close network client", e); - } - - log.debug("Shutdown of Kafka producer I/O thread has completed."); - } - - /** - * Run a single iteration of sending - * - * @param now - * The current POSIX time in milliseconds - */ - void run(long now) { - Set readyTopics = this.accumulator.ready(now); - // create produce requests - Map> batches = this.accumulator.drain(readyTopics, this.maxRequestSize, now); - if (guaranteeMessageOrder) { - // Mute all the partitions drained - for (List batchList : batches.values()) { - for (PubsubBatch batch : batchList) { - synchronized (accumulator) { - if (accumulator.isMutedTopic(batch.topic)) { - log.info("Another thread got same ordered topic before lock, removing.", batch.topic); - } else { - this.accumulator.muteTopic(batch.topic); - } - } - } - } - } - - List expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now); - // update sensors - for (PubsubBatch expiredBatch : expiredBatches) - this.sensors.recordErrors(expiredBatch.topic, expiredBatch.recordCount); - - sensors.updateProduceRequestMetrics(batches); - - if (!readyTopics.isEmpty()) { - log.trace("Topics with data ready to send: {}", readyTopics); - } - sendProduceRequests(batches, now); - } - - /** - * Start closing the sender (won't actually complete until all data is sent out) - */ - public void initiateClose() { - // Ensure accumulator is closed first to guarantee that no more appends are accepted after - // breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down. - this.accumulator.close(); - this.running = false; - this.executor.shutdown(); - } - - /** - * Closes the sender without sending out any pending messages. - */ - public void forceClose() { - this.forceClose = true; - initiateClose(); - } - - /** - * Complete or retry the given batch of records. - * - * @param batch The record batch - * @param error The error (or null if none) - * @param baseOffset The base offset assigned to the records if successful - * @param timestamp The timestamp returned by the broker for this batch - */ - private void completeBatch(PubsubBatch batch, Errors error, long baseOffset, long timestamp) { - if (error != Errors.NONE && canRetry(batch, error)) { - // retry - log.warn("Got error response on topic {}, retrying ({} attempts left). Error: {}", - batch.topic, - this.retries - batch.attempts - 1, - error); - batch.done(baseOffset, timestamp, error.exception()); - this.accumulator.reenqueue(batch, time.milliseconds()); - this.sensors.recordRetries(batch.topic, batch.recordCount); - } else { - RuntimeException exception; - if (error == Errors.TOPIC_AUTHORIZATION_FAILED) - exception = new TopicAuthorizationException(batch.topic); - else - exception = error.exception(); - // tell the user the result of their request - batch.done(baseOffset, timestamp, exception); - this.accumulator.deallocate(batch); - if (error != Errors.NONE) - this.sensors.recordErrors(batch.topic, batch.recordCount); - } - - // Unmute the completed partition. - if (guaranteeMessageOrder) - this.accumulator.unmuteTopic(batch.topic); - } - - /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed - */ - private boolean canRetry(PubsubBatch batch, Errors error) { - return batch.attempts < this.retries && error.exception() instanceof RetriableException; - } - - /** - * Transfer the record batches into a list of produce requests on a per-node basis - */ - private void sendProduceRequests(Map> collated, long now) { - for (Map.Entry> entry : collated.entrySet()) - sendProduceRequest(now, requestTimeout, entry.getValue()); - } - - /** - * Create a produce request from the given record batches - */ - private void sendProduceRequest(long sendTime, long timeout, List batches) { - for (PubsubBatch batch : batches) { - PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(batch.topic); - request.addMessages(PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(batch.records.buffer()))); - executor.submit(new ProduceRequestThread(sendTime, timeout, batch, request.build())); - log.trace("Sent produce request to topic {}", batch.topic); - } - } - - private class ProduceRequestThread implements Runnable { - private PublishRequest request; - private PubsubBatch batch; - private long timeout; - private long sendTime; - - public ProduceRequestThread(long sendTime, long timeout, PubsubBatch batch, PublishRequest request) { - this.timeout = timeout; - this.sendTime = sendTime; - this.request = request; - this.batch = batch; - } - - @Override - public void run() { - long receivedTime = time.milliseconds(); - ListenableFuture future = stub.publish(request); - while (true) { - try { - PublishResponse response = future.get(timeout, TimeUnit.MILLISECONDS); - log.trace("Receved produce response from topic {}", batch.topic); - String id = response.getMessageIds(0); - long offset = Long.valueOf(id); - completeBatch(batch, Errors.NONE, offset, receivedTime); - return; - } catch (InterruptedException e) { - log.warn("Accessing publish future was interrupted, retrying"); - } catch (TimeoutException e) { - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - return; - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof StatusRuntimeException) { - Status.Code code = ((StatusRuntimeException) cause).getStatus().getCode(); - switch (code) { - case ABORTED: - case CANCELLED: - case INTERNAL: - completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, receivedTime); - break; - case DATA_LOSS: - completeBatch(batch, Errors.CORRUPT_MESSAGE, -1L, receivedTime); - break; - case DEADLINE_EXCEEDED: - completeBatch(batch, Errors.REQUEST_TIMED_OUT, -1L, receivedTime); - break; - case ALREADY_EXISTS: - case OUT_OF_RANGE: - case INVALID_ARGUMENT: - completeBatch(batch, Errors.INVALID_REQUEST, -1L, receivedTime); - break; - case NOT_FOUND: - completeBatch(batch, Errors.INVALID_TOPIC_EXCEPTION, -1L, receivedTime); - break; - case RESOURCE_EXHAUSTED: - completeBatch(batch, Errors.RECORD_LIST_TOO_LARGE, -1L, receivedTime); - break; - case PERMISSION_DENIED: - case UNAUTHENTICATED: - completeBatch(batch, Errors.TOPIC_AUTHORIZATION_FAILED, -1L, receivedTime); - break; - case FAILED_PRECONDITION: - case UNAVAILABLE: - completeBatch(batch, Errors.BROKER_NOT_AVAILABLE, -1L, receivedTime); - break; - case UNIMPLEMENTED: - completeBatch(batch, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, -1L, receivedTime); - break; - default: - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - break; - } - } else { // Status is not StatusRuntimeException - completeBatch(batch, Errors.UNKNOWN, -1L, receivedTime); - } - return; - } - sensors.recordLatency(batch.topic, receivedTime - sendTime); - } - } - } - - private class PubsubSenderMetrics { - private final Metrics metrics; - public final Sensor retrySensor; - public final Sensor errorSensor; - public final Sensor queueTimeSensor; - public final Sensor requestTimeSensor; - public final Sensor recordsPerRequestSensor; - public final Sensor batchSizeSensor; - public final Sensor compressionRateSensor; - public final Sensor maxRecordSizeSensor; - public final Sensor produceThrottleTimeSensor; - - public PubsubSenderMetrics(Metrics metrics) { - this.metrics = metrics; - String metricGrpName = "producer-metrics"; - - this.batchSizeSensor = metrics.sensor("batch-size"); - MetricName m = metrics.metricName("batch-size-avg", metricGrpName, "The average number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Avg()); - m = metrics.metricName("batch-size-max", metricGrpName, "The max number of bytes sent per partition per-request."); - this.batchSizeSensor.add(m, new Max()); - - this.compressionRateSensor = metrics.sensor("compression-rate"); - m = metrics.metricName("compression-rate-avg", metricGrpName, "The average compression rate of record batches."); - this.compressionRateSensor.add(m, new Avg()); - - this.queueTimeSensor = metrics.sensor("queue-time"); - m = metrics.metricName("record-queue-time-avg", metricGrpName, "The average time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Avg()); - m = metrics.metricName("record-queue-time-max", metricGrpName, "The maximum time in ms record batches spent in the record accumulator."); - this.queueTimeSensor.add(m, new Max()); - - this.requestTimeSensor = metrics.sensor("request-time"); - m = metrics.metricName("request-latency-avg", metricGrpName, "The average request latency in ms"); - this.requestTimeSensor.add(m, new Avg()); - m = metrics.metricName("request-latency-max", metricGrpName, "The maximum request latency in ms"); - this.requestTimeSensor.add(m, new Max()); - - this.produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); - m = metrics.metricName("produce-throttle-time-avg", metricGrpName, "The average throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Avg()); - m = metrics.metricName("produce-throttle-time-max", metricGrpName, "The maximum throttle time in ms"); - this.produceThrottleTimeSensor.add(m, new Max()); - - this.recordsPerRequestSensor = metrics.sensor("records-per-request"); - m = metrics.metricName("record-send-rate", metricGrpName, "The average number of records sent per second."); - this.recordsPerRequestSensor.add(m, new Rate()); - m = metrics.metricName("records-per-request-avg", metricGrpName, "The average number of records per request."); - this.recordsPerRequestSensor.add(m, new Avg()); - - this.retrySensor = metrics.sensor("record-retries"); - m = metrics.metricName("record-retry-rate", metricGrpName, "The average per-second number of retried record sends"); - this.retrySensor.add(m, new Rate()); - - this.errorSensor = metrics.sensor("errors"); - m = metrics.metricName("record-error-rate", metricGrpName, "The average per-second number of record sends that resulted in errors"); - this.errorSensor.add(m, new Rate()); - - this.maxRecordSizeSensor = metrics.sensor("record-size-max"); - m = metrics.metricName("record-size-max", metricGrpName, "The maximum record size"); - this.maxRecordSizeSensor.add(m, new Max()); - m = metrics.metricName("record-size-avg", metricGrpName, "The average record size"); - this.maxRecordSizeSensor.add(m, new Avg()); - - m = metrics.metricName("requests-in-flight", metricGrpName, "The current number of in-flight requests awaiting a response."); - this.metrics.addMetric(m, new Measurable() { - public double measure(MetricConfig config, long now) { - return executor.getActiveCount(); - } - }); - } - - private void maybeRegisterTopicMetrics(String topic) { - // if one sensor of the metrics has been registered for the topic, - // then all other sensors should have been registered; and vice versa - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = this.metrics.getSensor(topicRecordsCountName); - if (topicRecordCount == null) { - Map metricTags = Collections.singletonMap("topic", topic); - String metricGrpName = "producer-topic-metrics"; - - topicRecordCount = this.metrics.sensor(topicRecordsCountName); - MetricName m = this.metrics.metricName("record-send-rate", metricGrpName, metricTags); - topicRecordCount.add(m, new Rate()); - - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = this.metrics.sensor(topicByteRateName); - m = this.metrics.metricName("byte-rate", metricGrpName, metricTags); - topicByteRate.add(m, new Rate()); - - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = this.metrics.sensor(topicCompressionRateName); - m = this.metrics.metricName("compression-rate", metricGrpName, metricTags); - topicCompressionRate.add(m, new Avg()); - - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.sensor(topicRetryName); - m = this.metrics.metricName("record-retry-rate", metricGrpName, metricTags); - topicRetrySensor.add(m, new Rate()); - - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.sensor(topicErrorName); - m = this.metrics.metricName("record-error-rate", metricGrpName, metricTags); - topicErrorSensor.add(m, new Rate()); - } - } - - public void updateProduceRequestMetrics(Map> batches) { - long now = time.milliseconds(); - for (List topicBatch : batches.values()) { - int records = 0; - for (PubsubBatch batch : topicBatch) { - // register all per-topic metrics at once - String topic = batch.topic; - maybeRegisterTopicMetrics(topic); - - // per-topic record send rate - String topicRecordsCountName = "topic." + topic + ".records-per-batch"; - Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName)); - topicRecordCount.record(batch.recordCount); - - // per-topic bytes send rate - String topicByteRateName = "topic." + topic + ".bytes"; - Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName)); - topicByteRate.record(batch.records.sizeInBytes()); - - // per-topic compression rate - String topicCompressionRateName = "topic." + topic + ".compression-rate"; - Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName)); - topicCompressionRate.record(batch.records.compressionRate()); - - // global metrics - this.batchSizeSensor.record(batch.records.sizeInBytes(), now); - this.queueTimeSensor.record(batch.drainedMs - batch.createdMs, now); - this.compressionRateSensor.record(batch.records.compressionRate()); - this.maxRecordSizeSensor.record(batch.maxRecordSize, now); - records += batch.recordCount; - } - this.recordsPerRequestSensor.record(records, now); - } - } - - public void recordRetries(String topic, int count) { - long now = time.milliseconds(); - this.retrySensor.record(count, now); - String topicRetryName = "topic." + topic + ".record-retries"; - Sensor topicRetrySensor = this.metrics.getSensor(topicRetryName); - if (topicRetrySensor != null) - topicRetrySensor.record(count, now); - } - - public void recordErrors(String topic, int count) { - long now = time.milliseconds(); - this.errorSensor.record(count, now); - String topicErrorName = "topic." + topic + ".record-errors"; - Sensor topicErrorSensor = this.metrics.getSensor(topicErrorName); - if (topicErrorSensor != null) - topicErrorSensor.record(count, now); - } - - public void recordLatency(String node, long latency) { - long now = time.milliseconds(); - this.requestTimeSensor.record(latency, now); - if (!node.isEmpty()) { - String nodeTimeName = "node-" + node + ".latency"; - Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); - if (nodeRequestTime != null) - nodeRequestTime.record(latency, now); - } - } - } -} From 03d8ab07d4efdbef992922de9290a36794d6fa23 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 22 Feb 2017 14:11:28 -0800 Subject: [PATCH 107/140] Implementing publisher/producer using grpc backend. --- .../com/google/pubsub/clients/ClientMain.java | 79 ++ .../clients/consumer/PubsubConsumer.java | 1157 ++++++++--------- .../clients/producer/PubsubProducer.java | 775 ++++------- .../com/google/pubsub/common/PubsubUtils.java | 46 + .../clients/producer/PubsubProducerTest.java | 11 +- .../producer/internals/MockPubsubServer.java | 67 - .../internals/PubsubAccumulatorTest.java | 412 ------ .../producer/internals/PubsubSenderTest.java | 210 --- 8 files changed, 906 insertions(+), 1851 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java new file mode 100644 index 00000000..5bba4d7c --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.pubsub.clients; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.PubsubProducer; +import org.apache.kafka.clients.producer.Producer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Properties; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Class to test simple features of PubsubProducer and PubsubConsumer. + */ +public class ClientMain { + + private static final Logger log = LoggerFactory.getLogger(ClientMain.class); + + public static void main(String[] args) throws Exception { + // going to set up the producer and its properties + String topic = args[0]; + String messageBody = args[1]; + + ClientMain main = new ClientMain(); + new Thread( + new Runnable() { + public void run() { + //main.subscriberExample(); + } + }) + .start(); + Thread.sleep(5000); + main.publisherExample(topic, messageBody); + } + + public void publisherExample(String topic, String messageBody) { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("project", "dataproc-kafka-test") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("acks", "all") + .put("batch.size", 1) + .put("linger.ms", 1) + .build() + ); + Producer publisher = new PubsubProducer<>(props); + + ProducerRecord msg = new ProducerRecord(topic, messageBody); + + publisher.send( + msg, + new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + log.error("Exception sending the message: " + exception.getMessage()); + } else { + log.info("Successfully sent message."); + } + } + } + ); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 72eb4413..e5abef92 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,14 +1,16 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ <<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java package com.google.kafka.cients.consumer; @@ -16,18 +18,17 @@ package com.google.pubsub.clients.consumer; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.ConsumerConfig; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.PollCondition; -import org.apache.kafka.clients.consumer.internals.Fetcher; -import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.PartitionAssignor; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -36,7 +37,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -71,608 +71,523 @@ public class PubsubConsumer implements Consumer { - private static final Logger log = LoggerFactor.getLogger(PubsubConsumer.class); - private static final long NO_CURRENT_THREAD = -1L; - private static final AtomicInteger CONSUMER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "pubsub.consumer"; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - - private final String clientId; - private final ConsumerCoordinator coordinator; - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Fetcher fetcher; - private final ConsumerInterceptors interceptors; - - private final Time time; - private final ConsumerNetworkClient client; - private final Metrics metrics; - private final SubscriptionState subscriptions; - private final Metadata metadata; // not sure if pubsub will have metadata - private final long retryBackoffMs; - private final long requestTimeoutMs; - private volatile boolean closed = false; - - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - private final AtomicInteger refcount = new AtomicInteger(0); - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings - * are documented here. Values can be - * either strings or objects of the appropriate type (for example a numeric configuration would accept either the - * string "42" or the integer 42). - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - */ - public PubsubConsumer(Map configs) { - this(configs, null, null); - } - - /** - * A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param configs The consumer configs - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - */ - public PubsubConsumer(Properties properties) { - this(properties, null, null); - } - - /** - * A consumer is instantiated by providing a {@link java.util.Properties} object as configuration, and a - * key and a value {@link Deserializer}. - *

- * Valid configuration strings are documented at {@link ConsumerConfig} - * - * @param properties The consumer configuration properties - * @param keyDeserializer The deserializer for key that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - * @param valueDeserializer The deserializer for value that implements {@link Deserializer}. The configure() method - * won't be called in the consumer when the deserializer is passed in directly. - */ - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } - - @SuppressWarnings("unchecked") - private PubsubConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - - } - - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ - public Set assignment() { - - } - - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ - public Set subscription() { - - } - - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - - } - - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ - @Override - public void subscribe(Collection topics) { - - } - - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - - } - - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ - public void unsubscribe() { - - } - - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ - @Override - public void assign(Collection partitions) { - - } - - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ - @Override - public ConsumerRecords poll(long timeout) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync() { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ - @Override - public void commitSync(final Map offsets) { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ - @Override - public void commitAsync() { - - } - - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(OffsetCommitCallback callback) { - - } - - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ - @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - - } - - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ - @Override - public void seek(TopicPartition partition, long offset) { - - } - - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ - public void seekToBeginning(Collection partitions) { - - } - - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ - public void seekToEnd(Collection partitions) { - - } - - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - public long position(TopicPartition partition) { - - } - - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public OffsetAndMetadata committed(TopicPartition partition) { - - } - - /** - * Get the metrics kept by the consumer - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); - } - - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public List partitionsFor(String topic) { - - } - - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ - @Override - public Map> listTopics() { - - } - - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ - @Override - public void pause(Collection partitions) { - - } - - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ - @Override - public void resume(Collection partitions) { - - } - - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ - @Override - public Set paused() { - - } - - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ - @Override - public Map offsetsForTimes(Map timestampsToSearch) { - - } - - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ - @Override - public Map beginningOffsets(Collection partitions) { - - } - - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ - @Override - public Map endOffsets(Collection partitions) { - - } - - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ - @Override - public void close() { - - } - - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ - public void close(long timeout, TimeUnit timeUnit) { - - } - - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ - @Override - public void wakeup() { - - } + public PubsubConsumer(Map configs) { + + } + + public PubsubConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + public PubsubConsumer(Properties properties) { + + } + + public PubsubConsumer(Properties properties, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + } + + /** + * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning + * partitions using {@link #assign(Collection)} then this will simply return the same partitions that + * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned + * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the + * process of getting reassigned). + * @return The set of partitions currently assigned to this consumer + */ + public Set assignment() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the current subscription. Will return the same topics used in the most recent call to + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. + * @return The set of topics currently subscribed to + */ + public Set subscription() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically + * assigned partitions. Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). Note that it is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that belong to a particular + * group and will trigger a rebalance operation if one of the following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ *

+ * When any of these events are triggered, the provided listener will be invoked first to indicate that + * the consumer's assignment has been revoked, and then again when the new assignment has been received. + * Note that this listener will immediately override any listener set in a previous call to subscribe. + * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics + * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. + * + * @param topics The list of topics to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which + * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer + * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets + * to be reset. You should also provide your own listener if you are doing your own offset + * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. + * + * @param topics The list of topics to subscribe to + * @throws IllegalArgumentException If topics is null or contains null or empty elements + */ + @Override + public void subscribe(Collection topics) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics + * existing at the time of check. + * + *

+ * As part of group management, the consumer will keep track of the list of consumers that + * belong to a particular group and will trigger a rebalance operation if one of the + * following events trigger - + *

    + *
  • Number of partitions change for any of the subscribed list of topics + *
  • Topic is created or deleted + *
  • An existing member of the consumer group dies + *
  • A new member is added to an existing consumer group via the join API + *
+ * + * @param pattern Pattern to subscribe to + * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the + * subscribed topics + * @throws IllegalArgumentException If pattern is null + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This + * also clears any partitions directly assigned through {@link #assign(Collection)}. + */ + public void unsubscribe() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment + * and will replace the previous assignment (if there is one). + * + * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * Manual topic assignment through this method does not use the consumer's group management + * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic + * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} + * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. + * + * @param partitions The list of partitions to assign this consumer + * @throws IllegalArgumentException If partitions is null or contains null or empty topics + */ + @Override + public void assign(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have + * subscribed to any topics or partitions before polling for data. + *

+ * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last + * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed + * offset for the subscribed list of partitions + * + * + * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. + * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. + * Must not be negative. + * @return map of topic to records since the last fetch for the subscribed list of topics and partitions + * + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of + * partitions is undefined or out of range and no offset reset policy has been configured + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed + * topics or to the configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or + * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * @throws java.lang.IllegalArgumentException if the timeout value is negative + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from + */ + @Override + public ConsumerRecords poll(long timeout) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is + * encountered (in which case it is thrown to the caller). + * + * @param offsets A map of offsets by partition with associated metadata + * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. + * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, + * or if there is an active group with the same groupId which is using group management. + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata + * is too large or if the committed offset is invalid). + */ + @Override + public void commitSync(final Map offsets) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. + * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} + */ + @Override + public void commitAsync() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. + *

+ * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after + * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + *

+ * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every + * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API + * should not be used. The committed offset should be the next message your application will consume, + * i.e. lastProcessedMessageOffset + 1. + *

+ * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback + * (if provided) or discarded. + * + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. + * @param callback Callback to invoke when the commit completes + */ + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API + * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets + */ + @Override + public void seek(TopicPartition partition, long offset) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the + * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the first offset for all of the currently assigned partitions. + */ + public void seekToBeginning(Collection partitions) { + + } + + /** + * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the + * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. + * If no partition is provided, seek to the final offset for all of the currently assigned partitions. + */ + public void seekToEnd(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + * + * @param partition The partition to get the position for + * @return The offset + * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for + * the partition + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public long position(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or + * another). This offset will be used as the position for the consumer in the event of a failure. + *

+ * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the + * consumer hasn't yet initialized its cache of committed offsets. + * + * @param partition The partition to check + * @return The last committed offset and metadata or null if there was no prior commit + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the + * configured groupId + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public OffsetAndMetadata committed(TopicPartition partition) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the metrics kept by the consumer + */ + public Map metrics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it + * does not already have any metadata about the given topic. + * + * @param topic The topic to get partition metadata for + * @return The list of partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a + * remote call to the server. + * @return The map of topics and its partitions + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before + * expiration of the configured request timeout + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + */ + public Map> listTopics() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return + * any records from these partitions until they have been resumed using {@link #resume(Collection)}. + * Note that this method does not affect partition subscription. In particular, it does not cause a group + * rebalance when automatic assignment is used. + * @param partitions The partitions which should be paused + */ + public void pause(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to + * {@link #poll(long)} will return records from these partitions if there are any to be fetched. + * If the partitions were not previously paused, this method is a no-op. + * @param partitions The partitions which should be resumed + */ + public void resume(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. + * + * @return The set of paused partitions + */ + public Set paused() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. + * + * This is a blocking call. The consumer does not have to be assigned the partitions. + * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null + * will be returned for that partition. + * + * Notice that this method may block indefinitely if the partition does not exist. + * + * @param timestampsToSearch the mapping from partition to the timestamp to look up. + * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater + * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no + * such message. + * @throws IllegalArgumentException if the target timestamp is negative. + * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up + * the offsets by timestamp. + */ + public Map offsetsForTimes(Map timestampsToSearch) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the first offset for the given partitions. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToBeginning(Collection) + * + * @param partitions the partitions to get the earliest offsets. + * @return The earliest available offsets for the given partitions + */ + public Map beginningOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming + * message, i.e. the offset of the last available message + 1. + *

+ * Notice that this method may block indefinitely if the partition does not exist. + * This method does not change the current consumer position of the partitions. + * + * @see #seekToEnd(Collection) + * + * @param partitions the partitions to get the end offsets. + * @return The end offsets for the given partitions. + */ + public Map endOffsets(Collection partitions) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. + * If auto-commit is enabled, this will commit the current offsets if possible within the default + * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} + * cannot be used to interrupt close. + * + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted + * before or while this function is called + */ + public void close() { + + } + + /** + * Tries to close the consumer cleanly within the specified timeout. This method waits up to + * timeout for the consumer to complete pending commits and leave the group. + * If auto-commit is enabled, this will commit the current offsets if possible within the + * timeout. If the consumer is unable to complete offset commits and gracefully leave the group + * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be + * used to interrupt close. + * + * @param timeout The maximum time to wait for consumer to close gracefully. The value must be + * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. + * @param timeUnit The time unit for the timeout + * @throws InterruptException If the thread is interrupted before or while this function is called + * @throws IllegalArgumentException If the timeout is negative. + */ + public void close(long timeout, TimeUnit timeUnit) { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. + * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. + * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. + */ + public void wakeup() { + throw new NotImplementedException("Not yet implemented"); + } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index c9c26b63..32856d8b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ + package com.google.pubsub.clients.producer; -import io.grpc.ManagedChannel; -import io.grpc.netty.NettyChannelBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PublishRequest; +import com.google.pubsub.v1.PublishResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.common.PubsubUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.ProducerConfig; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; -import com.google.kafka.clients.producer.internals.PubsubAccumulator; -import com.google.kafka.clients.producer.internals.PubsubSender; -import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.ApiException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -48,9 +56,10 @@ import org.slf4j.LoggerFactory; import java.net.SocketOptions; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,558 +76,252 @@ */ public class PubsubProducer implements Producer { - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final AtomicInteger PRODUCER_CLIENT_ID_SEQUENCE = new AtomicInteger(1); - private static final String JMX_PREFIX = "cps.producer"; - - private String clientId; - private final int maxRequestSize; - private final long totalMemorySize; - private final PubsubAccumulator accumulator; - private final PubsubSender sender; - private final Metrics metrics; - private final Thread ioThread; - private final CompressionType compressionType; - private final Sensor errors; - private final Time time; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final ProducerConfig producerConfig; - private final long maxBlockTimeMs; - private final int requestTimeoutMs; - private final ProducerInterceptors interceptors; - - public PubsubProducer(Map configs) { - this(new ProducerConfig(configs), null, null); + private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + + private PublisherFutureStub publisher; + private String project; + private Serializer keySerializer; + private Serializer valueSerializer; + private int batchSize; + private boolean isAcks; + private boolean closed = false; + private Map> perTopicBatch; + + public PubsubProducer(Map configs) { + this(new PubsubProducerConfig(configs), null, null); + } + + public PubsubProducer(Map configs, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + public PubsubProducer(Properties properties) { + this(new PubsubProducerConfig(properties), null, null); + } + + public PubsubProducer(Properties properties, Serializer keySerializer, + Serializer valueSerializer) { + this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + keySerializer, valueSerializer); + } + + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + try { + log.trace("Starting the Pubsub producer"); + publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + if (keySerializer == null) { + this.keySerializer = + configs.getConfiguredInstance( + PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.keySerializer.configure(configs.originals(), true); + } else { + configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getConfiguredInstance( + PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + this.valueSerializer.configure(configs.originals(), false); + } else { + configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + } catch (Exception e) { + throw new RuntimeException(e); } - public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); + project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + log.debug("Producer successfully initialized."); + } + + /** + * Send the given record asynchronously and return a future which will eventually contain the response information. + * + * @param record The record to send + * @return A future which will eventually contain the response information + */ + public Future send(ProducerRecord record) { + return send(record, null); + } + + /** + * Send a record and invoke the given callback when the record has been acknowledged by the server + */ + public Future send(ProducerRecord record, Callback callback) { + log.info("Received " + record.toString()); + if (closed) { + throw new RuntimeException("Publisher is closed"); } - public PubsubProducer(Properties properties) { - this(new ProducerConfig(properties), null, null); - } + String topic = record.topic(); + Map attributes = new HashMap<>(); - public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new ProducerConfig(ProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); + if (record.key() != null) { + byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - @SuppressWarnings({"unchecked", "deprecation"}) - private PubsubProducer(ProducerConfig config, Serializer keySerializer, Serializer valueSerializer) { - log.trace("Starting the Kafka producer"); - Map userProvidedConfigs = config.originals(); - this.producerConfig = config; - this.time = new SystemTime(); - - clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); - if (clientId.length() <= 0) - clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); - Map metricTags = new LinkedHashMap(); - metricTags.put("client-id", clientId); - MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) - .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) - .tags(metricTags); - List reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, - MetricsReporter.class); - reporters.add(new JmxReporter(JMX_PREFIX)); - this.metrics = new Metrics(metricConfig, reporters, time); - long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - if (keySerializer == null) { - this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.keySerializer.configure(config.originals(), true); - } else { - config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - if (valueSerializer == null) { - this.valueSerializer = config.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Serializer.class); - this.valueSerializer.configure(config.originals(), false); - } else { - config.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - - // load interceptors and make sure they get clientId - userProvidedConfigs.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = (List) (new ProducerConfig(userProvidedConfigs)).getConfiguredInstances(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, - ProducerInterceptor.class); - this.interceptors = interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - - this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); - this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); - /* check for user defined settings. - * If the BLOCK_ON_BUFFER_FULL is set to true,we do not honor METADATA_FETCH_TIMEOUT_CONFIG. - * This should be removed with release 0.9 when the deprecated configs are removed. - */ - if (userProvidedConfigs.containsKey(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG)) { - log.warn(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - boolean blockOnBufferFull = config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG); - if (blockOnBufferFull) { - this.maxBlockTimeMs = Long.MAX_VALUE; - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - } else if (userProvidedConfigs.containsKey(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG + " config is deprecated and will be removed soon. " + - "Please use " + ProducerConfig.MAX_BLOCK_MS_CONFIG); - this.maxBlockTimeMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); - } else { - this.maxBlockTimeMs = config.getLong(ProducerConfig.MAX_BLOCK_MS_CONFIG); - } - - /* check for user defined settings. - * If the TIME_OUT config is set use that for request timeout. - * This should be removed with release 0.9 - */ - if (userProvidedConfigs.containsKey(ProducerConfig.TIMEOUT_CONFIG)) { - log.warn(ProducerConfig.TIMEOUT_CONFIG + " config is deprecated and will be removed soon. Please use " + - ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.requestTimeoutMs = config.getInt(ProducerConfig.TIMEOUT_CONFIG); - } else { - this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - } - - this.accumulator = new PubsubAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), - this.totalMemorySize, - this.compressionType, - config.getLong(ProducerConfig.LINGER_MS_CONFIG), - retryBackoffMs, - metrics, - time); - - int sendBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (sendBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - sendBufferSize = SocketOptions.SO_SNDBUF; - int receiveBufferSize = config.getInt(ProducerConfig.SEND_BUFFER_CONFIG); - if (receiveBufferSize == Selectable.USE_DEFAULT_BUFFER_SIZE) - receiveBufferSize = SocketOptions.SO_RCVBUF; - - ManagedChannel channel = NettyChannelBuilder.forAddress("pubsub.googleapis.com", 443) - .flowControlWindow(sendBufferSize + receiveBufferSize) - .executor(new ThreadPoolExecutor(1, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), - config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), - TimeUnit.MILLISECONDS, new LinkedTransferQueue())).build(); - this.sender = new PubsubSender(channel, - this.accumulator, - config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) == 1, - config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), - config.getInt(ProducerConfig.RETRIES_CONFIG), - this.metrics, - new SystemTime(), - this.requestTimeoutMs); - String ioThreadName = "pubsub-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); - this.ioThread = new KafkaThread(ioThreadName, this.sender, true); - this.ioThread.start(); - - this.errors = this.metrics.sensor("errors"); - - config.logUnused(); - AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); - log.debug("Pubsub producer started"); + if (project == null) { + throw new RuntimeException("No project specified."); } - private static int parseAcks(String acksString) { - try { - return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim()); - } catch (NumberFormatException e) { - throw new ConfigException("Invalid configuration value for 'acks': " + acksString); - } + byte[] valueBytes = ByteString.EMPTY.toByteArray(); + if (record.value() != null) { + valueBytes = valueSerializer.serialize(topic, record.value()); } - /** - * Asynchronously send a record to a topic. Equivalent to send(record, null). - * See {@link #send(ProducerRecord, Callback)} for details. - */ - @Override - public Future send(ProducerRecord record) { - return send(record, null); + PubsubMessage message = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); + List batch = perTopicBatch.get(topic); + if (batch == null) { + batch = new ArrayList<>(batchSize); + perTopicBatch.put(topic, batch); } - - /** - * Asynchronously send a record to a topic and invoke the provided callback when the send has been acknowledged. - *

- * The send is asynchronous and this method will return immediately once the record has been stored in the buffer of - * records waiting to be sent. This allows sending many records in parallel without blocking to wait for the - * response after each one. - *

- * The result of the send is a {@link RecordMetadata} specifying the partition the record was sent to, the offset - * it was assigned and the timestamp of the record. If - * {@link org.apache.kafka.common.record.TimestampType#CREATE_TIME CreateTime} is used by the topic, the timestamp - * will be the user provided timestamp or the record send time if the user did not specify a timestamp for the - * record. If {@link org.apache.kafka.common.record.TimestampType#LOG_APPEND_TIME LogAppendTime} is used for the - * topic, the timestamp will be the Kafka broker local time when the message is appended. - *

- * Since the send call is asynchronous it returns a {@link java.util.concurrent.Future Future} for the - * {@link RecordMetadata} that will be assigned to this record. Invoking {@link java.util.concurrent.Future#get() - * get()} on this future will block until the associated request completes and then return the metadata for the record - * or throw any exception that occurred while sending the record. - *

- * If you want to simulate a simple blocking call you can call the get() method immediately: - * - * 

-     * {@code
-     * byte[] key = "key".getBytes();
-     * byte[] value = "value".getBytes();
-     * ProducerRecord record = new ProducerRecord("my-topic", key, value)
-     * producer.send(record).get();
-     * }
- *

- * Fully non-blocking usage can make use of the {@link Callback} parameter to provide a callback that - * will be invoked when the request is complete. - * - * 

-     * {@code
-     * ProducerRecord record = new ProducerRecord("the-topic", key, value);
-     * producer.send(myRecord,
-     *               new Callback() {
-     *                   public void onCompletion(RecordMetadata metadata, Exception e) {
-     *                       if(e != null) {
-     *                          e.printStackTrace();
-     *                       } else {
-     *                          System.out.println("The offset of the record we just sent is: " + metadata.offset());
-     *                       }
-     *                   }
-     *               });
-     * }
-     *
- * - * Callbacks for records being sent to the same partition are guaranteed to execute in order. That is, in the - * following example callback1 is guaranteed to execute before callback2: - * - * 
-     * {@code
-     * producer.send(new ProducerRecord(topic, partition, key1, value1), callback1);
-     * producer.send(new ProducerRecord(topic, partition, key2, value2), callback2);
-     * }
-     *
- *

- * Note that callbacks will generally execute in the I/O thread of the producer and so should be reasonably fast or - * they will delay the sending of messages from other threads. If you want to execute blocking or computationally - * expensive callbacks it is recommended to use your own {@link java.util.concurrent.Executor} in the callback body - * to parallelize processing. - * - * @param record The record to send - * @param callback A user-supplied callback to execute when the record has been acknowledged by the server (null - * indicates no callback) - * - * @throws InterruptException If the thread is interrupted while blocked - * @throws SerializationException If the key or value are not valid objects given the configured serializers - * @throws TimeoutException If the time taken for fetching metadata or allocating memory for the record has surpassed max.block.ms. - * @throws KafkaException If a Kafka related error occurs that does not belong to the public API exceptions. - * - */ - @Override - public Future send(ProducerRecord record, Callback callback) { - // intercept the record, which can be potentially modified; this method does not throw exceptions - ProducerRecord interceptedRecord = this.interceptors == null ? record : this.interceptors.onSend(record); - return doSend(interceptedRecord, callback); + batch.add(message); + if (batch.size() == batchSize) { + log.trace("Sending a batch of messages."); + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); + doSend(request, callback); } + return new PubsubFutureRecordMetadata(); + } + + private Future doSend(PublishRequest request, Callback callback) { + try { + ListenableFuture response = publisher.publish(request); + if (callback != null) { + if (isAcks) { + Futures.addCallback( + response, + new FutureCallback() { + public void onSuccess(PublishResponse response) { + perTopicBatch.clear(); + callback.onCompletion(null, null); + } - /** - * Implementation of asynchronously send a record to a topic. - */ - private Future doSend(ProducerRecord record, Callback callback) { - TopicPartition tp = null; - try { - byte[] serializedKey; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + - " specified in key.serializer"); - } - byte[] serializedValue; - try { - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (ClassCastException cce) { - throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + - " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + - " specified in value.serializer"); - } - - int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(serializedKey, serializedValue); - ensureValidRecordSize(serializedSize); - tp = new TopicPartition(record.topic(), 0); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - log.trace("Sending record {} with callback {} to topic {}.", record, callback, record.topic()); - // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = - this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); - PubsubAccumulator.RecordAppendResult result = accumulator.append(tp.topic(), timestamp, serializedKey, - serializedValue, interceptCallback, maxBlockTimeMs); - return result.future; - // handling exceptions and record the errors; - // for API exceptions return them in the future, - // for other exceptions throw directly - } catch (ApiException e) { - log.debug("Exception occurred during message send:", e); - if (callback != null) - callback.onCompletion(null, e); - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - return new FutureFailure(e); - } catch (InterruptedException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw new InterruptException(e); - } catch (BufferExhaustedException e) { - this.errors.record(); - this.metrics.sensor("buffer-exhausted-records").record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (KafkaException e) { - this.errors.record(); - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; - } catch (Exception e) { - // we notify interceptor about all exceptions, since onSend is called before anything else in this method - if (this.interceptors != null) - this.interceptors.onSendError(record, tp, e); - throw e; + public void onFailure(Throwable t) { + callback.onCompletion(null, new Exception(t)); + } + } + ); + } else { + perTopicBatch.clear(); + callback.onCompletion(null, null); } + } else { + response.get(); + perTopicBatch.clear(); + } + } catch (InterruptedException | ExecutionException e) { + return new FutureFailure(e); } - - /** - * Validate that the record size isn't too large - */ - private void ensureValidRecordSize(int size) { - if (size > this.maxRequestSize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the maximum request size you have configured with the " + - ProducerConfig.MAX_REQUEST_SIZE_CONFIG + - " configuration."); - if (size > this.totalMemorySize) - throw new RecordTooLargeException("The message is " + size + - " bytes when serialized which is larger than the total memory buffer you have configured with the " + - ProducerConfig.BUFFER_MEMORY_CONFIG + - " configuration."); + return new PubsubFutureRecordMetadata(); + } + + /** + * Flush any accumulated records from the producer. Blocks until all sends are complete. + */ + public void flush() { + throw new NotImplementedException("Not yet implemented"); + } + + /** + * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change + * over time so this list should not be cached. + */ + public List partitionsFor(String topic) { + throw new NotImplementedException("Partitions not supported"); + } + + /** + * Return a map of metrics maintained by the producer + */ + public Map metrics() { + throw new NotImplementedException("Metrics not supported."); + } + + /** + * Close this producer + */ + public void close() { + close(0, null); + } + + /** + * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the + * timeout, fail any pending send requests and force close the producer. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("Timout cannot be negative."); } - /** - * Invoking this method makes all buffered records immediately available to send (even if linger.ms is - * greater than 0) and blocks on the completion of the requests associated with these records. The post-condition - * of flush() is that any previously sent record will have completed (e.g. Future.isDone() == true). - * A request is considered completed when it is successfully acknowledged - * according to the acks configuration you have specified or else it results in an error. - *

- * Other threads can continue sending records while one thread is blocked waiting for a flush call to complete, - * however no guarantee is made about the completion of records sent after the flush call begins. - *

- * This method can be useful when consuming from some input system and producing into Kafka. The flush() call - * gives a convenient way to ensure all previously sent messages have actually completed. - *

- * This example shows how to consume from one Kafka topic and produce to another Kafka topic: - * 

-     * {@code
-     * for(ConsumerRecord record: consumer.poll(100))
-     *     producer.send(new ProducerRecord("my-topic", record.key(), record.value());
-     * producer.flush();
-     * consumer.commit();
-     * }
-     *
- * - * Note that the above example may drop records if the produce request fails. If we want to ensure that this does not occur - * we need to set retries=<large_number> in our config. - * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void flush() { - log.trace("Flushing accumulated records in producer."); - this.accumulator.beginFlush(); - try { - this.accumulator.awaitFlushCompletion(); - } catch (InterruptedException e) { - throw new InterruptException("Flush interrupted.", e); - } - } + log.debug("Closed producer"); + closed = true; + } - /** - * Get the partition metadata for the give topic. This can be used for custom partitioning. - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public List partitionsFor(String topic) { - List out = new ArrayList<>(1); - out.add(new PartitionInfo(topic, 0, null, null, null)); - return out; + /** Implementation of {@link Future}. */ + private static class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { + return false; } - /** - * Get the full set of internal metrics maintained by the producer. - */ - @Override - public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + public boolean isCancelled() { + return false; } - /** - * Close this producer. This method blocks until all previously sent requests complete. - * This method is equivalent to close(Long.MAX_VALUE, TimeUnit.MILLISECONDS). - *

- * If close() is called from {@link Callback}, a warning message will be logged and close(0, TimeUnit.MILLISECONDS) - * will be called instead. We do this because the sender thread would otherwise try to join itself and - * block forever. - *

- * - * @throws InterruptException If the thread is interrupted while blocked - */ - @Override - public void close() { - close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + public boolean isDone() { + return false; } - /** - * This method waits up to timeout for the producer to complete the sending of all incomplete requests. - *

- * If the producer is unable to complete all requests before the timeout expires, this method will fail - * any unsent and unacknowledged records immediately. - *

- * If invoked from within a {@link Callback} this method will not block and will be equivalent to - * close(0, TimeUnit.MILLISECONDS). This is done since no further sending will happen while - * blocking the I/O thread of the producer. - * - * @param timeout The maximum time to wait for producer to complete any pending requests. The value should be - * non-negative. Specifying a timeout of zero means do not wait for pending send requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted while blocked - * @throws IllegalArgumentException If the timeout is negative. - */ - @Override - public void close(long timeout, TimeUnit timeUnit) { - close(timeout, timeUnit, false); + public RecordMetadata get() throws InterruptedException, ExecutionException { + return null; } - private void close(long timeout, TimeUnit timeUnit, boolean swallowException) { - if (timeout < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - log.info("Closing the Kafka producer with timeoutMillis = {} ms.", timeUnit.toMillis(timeout)); - // this will keep track of the first encountered exception - AtomicReference firstException = new AtomicReference(); - boolean invokedFromCallback = Thread.currentThread() == this.ioThread; - if (timeout > 0) { - if (invokedFromCallback) { - log.warn("Overriding close timeout {} ms to 0 ms in order to prevent useless blocking due to self-join. " + - "This means you have incorrectly invoked close with a non-zero timeout from the producer call-back.", timeout); - } else { - // Try to close gracefully. - if (this.sender != null) - this.sender.initiateClose(); - if (this.ioThread != null) { - try { - this.ioThread.join(timeUnit.toMillis(timeout)); - } catch (InterruptedException t) { - firstException.compareAndSet(null, t); - log.error("Interrupted while joining ioThread", t); - } - } - } - } - - if (this.sender != null && this.ioThread != null && this.ioThread.isAlive()) { - log.info("Proceeding to force close the producer since pending requests could not be completed " + - "within timeout {} ms.", timeout); - this.sender.forceClose(); - // Only join the sender thread when not calling from callback. - if (!invokedFromCallback) { - try { - this.ioThread.join(); - } catch (InterruptedException e) { - firstException.compareAndSet(null, e); - } - } - } - - ClientUtils.closeQuietly(interceptors, "producer interceptors", firstException); - ClientUtils.closeQuietly(metrics, "producer metrics", firstException); - ClientUtils.closeQuietly(keySerializer, "producer keySerializer", firstException); - ClientUtils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); - AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId); - log.debug("The Kafka producer has closed."); - if (firstException.get() != null && !swallowException) - throw new KafkaException("Failed to close kafka producer", firstException.get()); + public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { + return null; } + } - private static class FutureFailure implements Future { - - private final ExecutionException exception; - - public FutureFailure(Exception exception) { - this.exception = new ExecutionException(exception); - } - - @Override - public boolean cancel(boolean interrupt) { - return false; - } - - @Override - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ + private static class FutureFailure implements Future { + private final ExecutionException exception; - @Override - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } + public FutureFailure(Exception e) { + this.exception = new ExecutionException(e); + } - @Override - public boolean isCancelled() { - return false; - } + public boolean cancel(boolean interrupt) { + return false; + } - @Override - public boolean isDone() { - return true; - } + public RecordMetadata get() throws ExecutionException { + throw this.exception; + } + public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { + throw this.exception; } - /** - * A callback called when producer request is complete. It in turn calls user-supplied callback (if given) and - * notifies producer interceptors about the request completion. - */ - private static class InterceptorCallback implements Callback { - private final Callback userCallback; - private final ProducerInterceptors interceptors; - private final TopicPartition tp; - - public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { - this.userCallback = userCallback; - this.interceptors = interceptors; - this.tp = tp; - } + public boolean isCancelled() { + return false; + } - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (this.interceptors != null) { - if (metadata == null) { - this.interceptors.onAcknowledgement(new RecordMetadata(tp, -1, -1, Record.NO_TIMESTAMP, -1, -1, -1), - exception); - } else { - this.interceptors.onAcknowledgement(metadata, exception); - } - } - if (this.userCallback != null) - this.userCallback.onCompletion(metadata, exception); - } + public boolean isDone() { + return true; } + } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java new file mode 100644 index 00000000..10a5599e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptors; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; +import io.grpc.netty.NegotiationType; +import io.grpc.netty.NettyChannelBuilder; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; + +public class PubsubUtils { + + private static final String ENDPOINT = "pubsub.googleapis.com"; + private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); + + public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; + public static final String KEY_ATTRIBUTE = "key"; + public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; + + public static Channel createChannel() throws Exception { + final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); + final ClientAuthInterceptor interceptor = + new ClientAuthInterceptor( + GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), + Executors.newCachedThreadPool()); + return ClientInterceptors.intercept(channelImpl, interceptor); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index fd8e96b4..2e438413 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -16,7 +16,7 @@ */ package com.google.kafka.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; +/*import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,13 +32,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; +import java.util.Properties;*/ -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") +//@RunWith(PowerMockRunner.class) +//@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - @Test + /* @Test public void testSerializerClose() throws Exception { Map configs = new HashMap<>(); configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); @@ -110,4 +110,5 @@ public void testInvalidSocketReceiveBufferSize() throws Exception { config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); } + */ } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java deleted file mode 100644 index a4b2ce53..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/MockPubsubServer.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import io.grpc.stub.StreamObserver; -import java.util.LinkedList; -import java.util.Queue; - -public class MockPubsubServer extends PublisherGrpc.PublisherImplBase { - private Queue> responseList; - - public MockPubsubServer() { - responseList = new LinkedList<>(); - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - responseList.add(responseObserver); - } - - public int inFlightCount() { - return responseList.size(); - } - - public void respond(PublishResponse response) { - StreamObserver stream = responseList.poll(); - stream.onNext(response); - stream.onCompleted(); - } - - public void disconnect() { - for (int i = 0; i < 100; i++) { - if (responseList.isEmpty()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { } // not an issue, ignore - } - } - StreamObserver stream = responseList.poll(); - stream.onCompleted(); - } - - public boolean listen(int messagesExpected, long waitInMillis) { - for (int i = 0; i < waitInMillis / 50; i++) { - if (responseList.size() == messagesExpected) { - return true; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - return false; - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java deleted file mode 100644 index fe241b0c..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubAccumulatorTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.record.LogEntry; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.SystemTime; -import org.junit.After; -import org.junit.Test; - -public class PubsubAccumulatorTest { - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private SystemTime systemTime = new SystemTime(); - private byte[] key = "key".getBytes(); - private byte[] value = "value".getBytes(); - private int msgSize = Records.LOG_OVERHEAD + Record.recordSize(key, value); - private Metrics metrics = new Metrics(time); - private final long maxBlockTimeMs = 1000; - - @After - public void teardown() { - this.metrics.close(); - } - - @Test - public void testFull() throws Exception { - long now = time.milliseconds(); - int batchSize = 1024; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * batchSize, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = batchSize / msgSize; - for (int i = 0; i < appends; i++) { - // append to the first batch - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque batches = accum.batches().get(topic); - assertEquals(1, batches.size()); - assertTrue(batches.peekFirst().records.isWritable()); - assertEquals("No topics should be ready.", 0, accum.ready(now).size()); - } - - // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Deque allBatches = accum.batches().get(topic); - assertEquals(2, allBatches.size()); - Iterator batchesIterator = allBatches.iterator(); - assertFalse(batchesIterator.next().records.isWritable()); - assertTrue(batchesIterator.next().records.isWritable()); - assertEquals("Topic should be ready.", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - for (int i = 0; i < appends; i++) { - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - } - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testAppendLarge() throws Exception { - int batchSize = 512; - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - accum.append(topic, 0L, key, new byte[2 * batchSize], null, maxBlockTimeMs); - assertEquals("Topic should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - } - - @Test - public void testLinger() throws Exception { - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready", 0, accum.ready(time.milliseconds()).size()); - time.sleep(10); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - List batches = accum.drain(Collections.singleton(topic), Integer.MAX_VALUE, 0).get(topic); - assertEquals(1, batches.size()); - PubsubBatch batch = batches.get(0); - - Iterator iter = batch.records.iterator(); - LogEntry entry = iter.next(); - assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key()); - assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value()); - assertFalse("No more records", iter.hasNext()); - } - - @Test - public void testPartialDrain() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10L, 100L, metrics, time); - int appends = 1024 / msgSize + 1; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } - assertEquals("Partition's leader should be ready", Collections.singleton(topic), accum.ready(time.milliseconds())); - - List batches = accum.drain(Collections.singleton(topic), 1024, 0).get(topic); - assertEquals("But due to size bound only one partition should have been retrieved", 1, batches.size()); - } - - @SuppressWarnings("unused") - @Test - public void testStressfulSituation() throws Exception { - final int numThreads = 5; - final int msgs = 10000; - final int numParts = 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time); - List threads = new ArrayList(); - for (int i = 0; i < numThreads; i++) { - threads.add(new Thread() { - public void run() { - for (int i = 0; i < msgs; i++) { - try { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }); - } - for (Thread t : threads) - t.start(); - int read = 0; - long now = time.milliseconds(); - while (read < numThreads * msgs) { - Set readyTopics = accum.ready(now); - List batches = accum.drain(readyTopics, 5 * 1024, 0).get(topic); - if (batches != null) { - for (PubsubBatch batch : batches) { - for (LogEntry entry : batch.records) - read++; - accum.deallocate(batch); - } - } - } - - for (Thread t : threads) - t.join(); - } - - @Test - public void testNextReadyCheckDelay() throws Exception { - // Next check time will use lingerMs since this test won't trigger any retries/backoff - long lingerMs = 10L; - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - // Just short of going over the limit so we trigger linger time - int appends = 512 / msgSize; // Gets through 2 attempted sends without filling batch - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - time.sleep(lingerMs / 2); - - for (int i = 0; i < appends; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyNodes.size()); - - // Add enough to make data sendable immediately - for (int i = 0; i < appends + 1; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyNodes = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyNodes); - } - - @Test - public void testRetryBackoff() throws Exception { - long lingerMs = Long.MAX_VALUE / 4; - long retryBackoffMs = Long.MAX_VALUE / 2; - final PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - - long now = time.milliseconds(); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - Set readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic1 should be ready", Collections.singleton(topic), readyTopics); - Map> batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + lingerMs + 1); - assertEquals("Topic1 should be the only ready topic.", 1, batches.size()); - assertEquals("There should only be one batch drained.", 1, batches.get(topic).size()); - - // Reenqueue the batch - now = time.milliseconds(); - accum.reenqueue(batches.get(topic).get(0), now); - - // Put another message into accumulator - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - readyTopics = accum.ready(now + lingerMs + 1); - assertEquals("Topic should not be ready due to backoff", 0, readyTopics.size()); - - // topic though backoff, should drain both batches - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the first batch.", 1, batches.get(topic).size()); - readyTopics = accum.ready(now + retryBackoffMs + 1); - batches = accum.drain(readyTopics, Integer.MAX_VALUE, now + retryBackoffMs + 1); - assertEquals("Topic should drain the second batch.", 1, batches.get(topic).size()); - } - - @Test - public void testFlush() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); // Fill almost 1 complete batch - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.beginFlush(); - readyTopics = accum.ready(time.milliseconds()); - - // drain and deallocate all batches - Map> results = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - for (List batches: results.values()) - for (PubsubBatch batch: batches) - accum.deallocate(batch); - - // should be complete with no unsent records. - accum.awaitFlushCompletion(); - assertFalse(accum.hasUnsent()); - } - - private void delayedInterrupt(final Thread thread, final long delayMs) { - Thread t = new Thread() { - public void run() { - systemTime.sleep(delayMs); - thread.interrupt(); - } - }; - t.start(); - } - - @Test - public void testAwaitFlushComplete() throws Exception { - PubsubAccumulator accum = new PubsubAccumulator(4 * 1024, 64 * 1024, CompressionType.NONE, Long.MAX_VALUE, 100L, metrics, time); - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - - accum.beginFlush(); - assertTrue(accum.flushInProgress()); - delayedInterrupt(Thread.currentThread(), 1000L); - try { - accum.awaitFlushCompletion(); - fail("awaitFlushCompletion should throw InterruptException"); - } catch (InterruptedException e) { - assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress()); - } - } - - @Test - public void testAbortIncompleteBatches() throws Exception { - long lingerMs = Long.MAX_VALUE; - int batchSize = 4 * 1024; - final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0); - final PubsubAccumulator accum = new PubsubAccumulator(batchSize, 64 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time); - class TestCallback implements Callback { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - assertTrue(exception.getMessage().equals("Producer is closed forcefully.")); - numExceptionReceivedInCallback.incrementAndGet(); - } - } - int attempts = batchSize / msgSize; - for (int i = 0; i < attempts; i++) - accum.append(topic, 0L, key, value, new TestCallback(), maxBlockTimeMs); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No topics should be ready.", 0, readyTopics.size()); - - accum.abortIncompleteBatches(); - assertEquals(numExceptionReceivedInCallback.get(), attempts); - assertFalse(accum.hasUnsent()); - - } - - @Test - public void testExpiredBatches() throws InterruptedException { - long retryBackoffMs = 100L; - long lingerMs = 3000L; - int batchSize = 1024; - int requestTimeout = 60; - - PubsubAccumulator accum = new PubsubAccumulator(batchSize, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time); - int appends = batchSize / msgSize; - - // Test batches not in retry - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - } - // Make the batches ready due to batch full - accum.append(topic, 0L, key, value, null, 0); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - // Advance the clock to expire the batch. - time.sleep(requestTimeout + 1); - accum.muteTopic(topic); - List expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Advance the clock to make the next batch ready due to linger.ms - time.sleep(lingerMs); - assertEquals("Topic should be ready", Collections.singleton(topic), readyTopics); - time.sleep(requestTimeout + 1); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted", 1, expiredBatches.size()); - assertEquals("No topics should be ready.", 0, accum.ready(time.milliseconds()).size()); - - // Test batches in retry. - // Create a retried batch - accum.append(topic, 0L, key, value, null, 0); - time.sleep(lingerMs); - readyTopics = accum.ready(time.milliseconds()); - assertEquals("Our partition's leader should be ready", Collections.singleton(topic), readyTopics); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("There should be only one batch.", drained.get(topic).size(), 1); - time.sleep(1000L); - accum.reenqueue(drained.get(topic).get(0), time.milliseconds()); - - // test expiration. - time.sleep(requestTimeout + retryBackoffMs); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired.", 0, expiredBatches.size()); - time.sleep(1L); - - accum.muteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should not be expired when the topic is muted", 0, expiredBatches.size()); - - accum.unmuteTopic(topic); - expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds()); - assertEquals("The batch should be expired when the topic is not muted.", 1, expiredBatches.size()); - } - - @Test - public void testMutedPartitions() throws InterruptedException { - long now = time.milliseconds(); - PubsubAccumulator accum = new PubsubAccumulator(1024, 10 * 1024, CompressionType.NONE, 10, 100L, metrics, time); - int appends = 1024 / msgSize; - for (int i = 0; i < appends; i++) { - accum.append(topic, 0L, key, value, null, maxBlockTimeMs); - assertEquals("No partitions should be ready.", 0, accum.ready(now).size()); - } - time.sleep(2000); - - // Test ready with muted partition - accum.muteTopic(topic); - Set readyTopics = accum.ready(time.milliseconds()); - assertEquals("No node should be ready", 0, readyTopics.size()); - - // Test ready without muted partition - accum.unmuteTopic(topic); - readyTopics = accum.ready(time.milliseconds()); - assertTrue("The batch should be ready", readyTopics.size() > 0); - - // Test drain with muted partition - accum.muteTopic(topic); - Map> drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertEquals("No batch should have been drained", 0, drained.get(topic).size()); - - // Test drain without muted partition. - accum.unmuteTopic(topic); - drained = accum.drain(readyTopics, Integer.MAX_VALUE, time.milliseconds()); - assertTrue("The batch should have been drained.", drained.get(topic).size() > 0); - } -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java deleted file mode 100644 index 15256379..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/internals/PubsubSenderTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.google.kafka.clients.producer.internals; - -import com.google.pubsub.v1.PublishResponse; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.MockTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PubsubSenderTest { - - private static final int MAX_REQUEST_SIZE = 1024 * 1024; - private static final short ACKS_ALL = -1; - private static final int MAX_RETRIES = 0; - private static final String CLIENT_ID = "clientId"; - private static final String METRIC_GROUP = "producer-metrics"; - private static final double EPS = 0.0001; - private static final int MAX_BLOCK_TIMEOUT = 1000; - private static final int REQUEST_TIMEOUT = 10000; - - private String topic = "test-topic"; - private MockTime time = new MockTime(); - private int batchSize = 16 * 1024; - private Metrics metrics = null; - private PubsubAccumulator accumulator = null; - - @Rule - public Timeout globalTimeout = Timeout.seconds(15); - - @Before - public void setup() { - Map metricTags = new LinkedHashMap<>(); - metricTags.put("client-id", CLIENT_ID); - MetricConfig metricConfig = new MetricConfig().tags(metricTags); - metrics = new Metrics(metricConfig, time); - accumulator = new PubsubAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time); - } - - @After - public void tearDown() { - this.metrics.close(); - } - - @Test - public void testSimple() throws Exception { - MockPubsubServer server = newServer("testSimple"); - PubsubSender sender = newSender("testSimple", MAX_RETRIES); - long offset = 32; - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // Sends produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - assertNotNull("Request should be completed", future.get()); - waitForUnmute(topic, 1000); - } - - @Test - public void testRetries() throws Exception { - int maxRetries = 1; - MockPubsubServer server = newServer("testRetries"); - PubsubSender sender = newSender("testRetries", maxRetries); - // do a successful retry - Future future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - assertEquals("All requests completed.", 0, server.inFlightCount()); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - sender.run(time.milliseconds()); // send second produce request - assertTrue("Server should receive request..", server.listen(1, 1000)); - long offset = 32; - server.respond(PublishResponse.newBuilder().addMessageIds(Long.toString(offset)).build()); - assertEquals("All requests completed.", 0, (long) server.inFlightCount()); - eventualReturn(future, 1000); - assertEquals(offset, future.get().offset()); - waitForUnmute(topic, 1000); - - // do an unsuccessful retry - future = accumulator.append(topic, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; - for (int i = 0; i < maxRetries + 1; i++) { - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - server.disconnect(); - completedWithError(future, Errors.NETWORK_EXCEPTION); - waitForUnmute(topic, 1000); - } - sender.run(time.milliseconds()); - assertEquals("Retry request should be received.", 0, server.inFlightCount()); - waitForUnmute(topic, 1000); - } - - @Test - public void testSendInOrder() throws Exception { - PubsubSender sender = newSender("testSendInOrder", MAX_RETRIES); - MockPubsubServer server = newServer("testSendInOrder"); - - // Send the first message. - accumulator.append(topic, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); - sender.run(time.milliseconds()); // send produce request - assertTrue("Server should receive request.", server.listen(1, 1000)); - - time.sleep(900); - // Now send another message - accumulator.append(topic, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); - - // Sender should not send second message before first is returned - sender.run(time.milliseconds()); - assertTrue("Server expects only one request.", server.listen(1, 1000)); - } - - private void completedWithError(Future future, Errors error) throws Exception { - try { - future.get(); - fail("Should have thrown an exception."); - } catch (ExecutionException e) { - assertEquals(error.exception().getClass(), e.getCause().getClass()); - } - } - - private void eventualReturn(Future future, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - try { - if (future.get() != null) { - return; - } else { - break; - } - } catch (ExecutionException | InterruptedException e) { } // Ignore and wait a turn - try { - Thread.sleep(50); - } catch (InterruptedException e) { - i--; // Not a big deal to be interrupted, just go another time through the loop - } - } - fail("Should have received a non-null result from future without exception"); - } - - private void waitForUnmute(String topic, long waitMillis) { - for (int i = 0; i < waitMillis / 50; i++) { - if (!accumulator.isMutedTopic(topic)) { - return; - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { } - } - fail(topic + " was never unmuted."); - } - - private PubsubSender newSender(String channelName, int retries) { - return new PubsubSender(InProcessChannelBuilder.forName(channelName).directExecutor().build(), - this.accumulator, - true, - MAX_REQUEST_SIZE, - retries, - metrics, - time, - REQUEST_TIMEOUT); - } - - private MockPubsubServer newServer(String channelName) { - MockPubsubServer out = new MockPubsubServer(); - try { - InProcessServerBuilder.forName(channelName).directExecutor().addService(out).build().start(); - } catch (IOException e) { - return null; - } - return out; - } - -// private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) { -// ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP); -// Map partResp = Collections.singletonMap(tp, resp); -// return new ProduceResponse(partResp, throttleTimeMs); -// } - -} From f9f19cde91f30e67099fee0dc34eb6f253867121 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Feb 2017 14:33:09 -0800 Subject: [PATCH 108/140] Add details to simplified producer --- .../clients/producer/PubsubProducer.java | 57 ++++++++----------- .../internals/PubsubFutureRecordMetadata.java | 1 + 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 32856d8b..54a31053 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -32,10 +32,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.JmxReporter; @@ -85,7 +87,8 @@ public class PubsubProducer implements Producer { private int batchSize; private boolean isAcks; private boolean closed = false; - private Map> perTopicBatch; + private Map> perTopicBatches; + private final int maxRequestSize; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -136,7 +139,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - perTopicBatch = Collections.synchronizedMap(new HashMap<>()); + maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + perTopicBatches = Collections.synchronizedMap(new HashMap<>()); log.debug("Producer successfully initialized."); } @@ -162,8 +166,9 @@ public Future send(ProducerRecord record, Callback callbac String topic = record.topic(); Map attributes = new HashMap<>(); + byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - byte[] serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } @@ -176,15 +181,17 @@ public Future send(ProducerRecord record, Callback callbac valueBytes = valueSerializer.serialize(topic, record.value()); } + checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatch.get(topic); + List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); - perTopicBatch.put(topic, batch); + perTopicBatches.put(topic, batch); } batch.add(message); if (batch.size() == batchSize) { @@ -196,7 +203,7 @@ public Future send(ProducerRecord record, Callback callbac .build(); doSend(request, callback); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); } private Future doSend(PublishRequest request, Callback callback) { @@ -208,7 +215,7 @@ private Future doSend(PublishRequest request, Callback callback) response, new FutureCallback() { public void onSuccess(PublishResponse response) { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } @@ -218,17 +225,24 @@ public void onFailure(Throwable t) { } ); } else { - perTopicBatch.clear(); + perTopicBatches.clear(); callback.onCompletion(null, null); } } else { response.get(); - perTopicBatch.clear(); + perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return new PubsubFutureRecordMetadata(); + return null; //new FutureRecordMetadata(); + } + + private void checkRecordSize(int size) { + if (size > this.maxRequestSize) { + throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + + " configured"); + } } /** @@ -273,29 +287,6 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - /** Implementation of {@link Future}. */ - private static class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { - return false; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return false; - } - - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return null; - } - } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 4c16e71f..66d0603f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -22,6 +22,7 @@ import org.apache.kafka.clients.producer.RecordMetadata; public class PubsubFutureRecordMetadata implements Future { + public boolean cancel(boolean b) { return false; } From f29c433be041e52c434eda6c1a5214aba575f594 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 27 Feb 2017 15:14:47 -0800 Subject: [PATCH 109/140] Producer implemented; close and flush newly implemented --- pubsub-mapped-api/pom.xml | 5 + .../com/google/pubsub/clients/ClientMain.java | 79 ---------------- .../google/pubsub/clients/ProducerThread.java | 5 +- .../pubsub/clients/ProducerThreadPool.java | 2 +- .../clients/producer/PubsubProducer.java | 46 +++++++-- .../internals/PubsubFutureRecordMetadata.java | 5 - .../pubsub/common/PubsubChannelUtil.java | 21 ++--- .../com/google/pubsub/common/PubsubUtils.java | 46 --------- .../clients/producer/PubsubProducerTest.java | 94 ++++--------------- 9 files changed, 70 insertions(+), 233 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 9dbf6b74..60f4ca73 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -88,6 +88,11 @@ mockito-all 2.0.2-beta + + com.google.api-client + google-api-client + 1.22.0 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java deleted file mode 100644 index 5bba4d7c..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ClientMain.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.pubsub.clients; - -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.PubsubProducer; -import org.apache.kafka.clients.producer.Producer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Class to test simple features of PubsubProducer and PubsubConsumer. - */ -public class ClientMain { - - private static final Logger log = LoggerFactory.getLogger(ClientMain.class); - - public static void main(String[] args) throws Exception { - // going to set up the producer and its properties - String topic = args[0]; - String messageBody = args[1]; - - ClientMain main = new ClientMain(); - new Thread( - new Runnable() { - public void run() { - //main.subscriberExample(); - } - }) - .start(); - Thread.sleep(5000); - main.publisherExample(topic, messageBody); - } - - public void publisherExample(String topic, String messageBody) { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", 1) - .put("linger.ms", 1) - .build() - ); - Producer publisher = new PubsubProducer<>(props); - - ProducerRecord msg = new ProducerRecord(topic, messageBody); - - publisher.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message."); - } - } - } - ); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 4f3e3427..87a6afc6 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,7 +1,7 @@ package com.google.pubsub.clients; +import com.google.api.client.googleapis.MethodOverride.Builder; import com.google.pubsub.clients.producer.PubsubProducer; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; @@ -24,6 +24,8 @@ public ProducerThread(String s, Properties props, String topic) throws IOExcepti .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .build(); + this.command = s; + this.producer = new PubsubProducer<>(props); this.topic = topic; } @@ -50,6 +52,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { } ); } + Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 36ef8716..5533c79f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -16,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) throws IOException { + public static void main(String[] args) { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 54a31053..291ba7a9 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -24,7 +24,9 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubUtils; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; +import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; @@ -33,6 +35,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; +import org.apache.kafka.clients.producer.internals.ProduceRequestResult; +import org.apache.kafka.clients.producer.internals.RecordAccumulator; +import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -89,6 +95,8 @@ public class PubsubProducer implements Producer { private boolean closed = false; private Map> perTopicBatches; private final int maxRequestSize; + private final Time time; + private PubsubChannelUtil channelUtil; public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -113,7 +121,9 @@ public PubsubProducer(Properties properties, Serializer keySerializer, private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - publisher = PublisherGrpc.newFutureStub(PubsubUtils.createChannel()); + this.time = new SystemTime(); + channelUtil = new PubsubChannelUtil(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -141,6 +151,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + + String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -169,7 +181,7 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(PubsubUtils.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -194,19 +206,24 @@ public Future send(ProducerRecord record, Callback callbac perTopicBatches.put(topic, batch); } batch.add(message); - if (batch.size() == batchSize) { + + long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); + RecordAccumulator.RecordAppendResult result = new RecordAppendResult( + new FutureRecordMetadata(new ProduceRequestResult(), 0, + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(PubsubUtils.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback); + doSend(request, callback, result); } - return null; //new FutureRecordMetadata(); + return result.future; } - private Future doSend(PublishRequest request, Callback callback) { + private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -235,7 +252,7 @@ public void onFailure(Throwable t) { } catch (InterruptedException | ExecutionException e) { return new FutureFailure(e); } - return null; //new FutureRecordMetadata(); + return result.future; } private void checkRecordSize(int size) { @@ -249,7 +266,15 @@ private void checkRecordSize(int size) { * Flush any accumulated records from the producer. Blocks until all sends are complete. */ public void flush() { - throw new NotImplementedException("Not yet implemented"); + log.debug("Flushing..."); + for (String topic : perTopicBatches.keySet()) { + PublishRequest request = + PublishRequest.newBuilder() + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); + doSend(request, null, null); + } } /** @@ -283,6 +308,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timout cannot be negative."); } + channelUtil.closeChannel(); log.debug("Closed producer"); closed = true; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java index 66d0603f..2b2f18e8 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/internals/PubsubFutureRecordMetadata.java @@ -22,7 +22,6 @@ import org.apache.kafka.clients.producer.RecordMetadata; public class PubsubFutureRecordMetadata implements Future { - public boolean cancel(boolean b) { return false; } @@ -35,10 +34,6 @@ public boolean isDone() { return false; } - public RecordMetadata get() throws InterruptedException, ExecutionException { - return null; - } - public RecordMetadata get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return null; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 9088d5bf..1991aa38 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,24 +16,22 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; +import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; +import io.grpc.auth.ClientAuthInterceptor; +import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.concurrent.Executors; -/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { - private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -43,15 +41,8 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; - /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() { - GoogleCredentials credentials; - try { - credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); - } catch (IOException exception) { - log.error("Exception occurred: " + exception.getMessage()); - return; - } + public PubsubChannelUtil() throws IOException { + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java deleted file mode 100644 index 10a5599e..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.pubsub.common; - -import com.google.auth.oauth2.GoogleCredentials; -import io.grpc.Channel; -import io.grpc.ClientInterceptors; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; -import io.grpc.netty.NegotiationType; -import io.grpc.netty.NettyChannelBuilder; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.Executors; - -public class PubsubUtils { - - private static final String ENDPOINT = "pubsub.googleapis.com"; - private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); - - public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; - public static final String KEY_ATTRIBUTE = "key"; - public static final String NO_PROJECT_ERROR = "No project specified. Use AdapterClient to specify project."; - - public static Channel createChannel() throws Exception { - final ManagedChannelImpl channelImpl = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - final ClientAuthInterceptor interceptor = - new ClientAuthInterceptor( - GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE), - Executors.newCachedThreadPool()); - return ClientInterceptors.intercept(channelImpl, interceptor); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 2e438413..98f6b889 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.kafka.clients.producer; +/*package com.google.kafka.clients.producer; -/*import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -32,83 +32,25 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties;*/ +import java.util.Properties; -//@RunWith(PowerMockRunner.class) -//@PowerMockIgnore("javax.management.*") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { - /* @Test - public void testSerializerClose() throws Exception { - Map configs = new HashMap<>(); - configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); - configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - configs.put(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL); - final int oldInitCount = MockSerializer.INIT_COUNT.get(); - final int oldCloseCount = MockSerializer.CLOSE_COUNT.get(); + @Test + public void testConstructorWithSerializers() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); + } - PubsubProducer producer = new PubsubProducer( - configs, new MockSerializer(), new MockSerializer()); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get()); + @Test(expected = ConfigException.class) + public void testNoSerializerProvided() { + Properties props = new Properties(); + props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props); + } - producer.close(); - Assert.assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get()); - Assert.assertEquals(oldCloseCount + 2, MockSerializer.CLOSE_COUNT.get()); - } - @Test - public void testInterceptorConstructClose() throws Exception { - try { - Properties props = new Properties(); - // test with client ID assigned by PubsubProducer - props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName()); - props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something"); - - PubsubProducer producer = new PubsubProducer( - props, new StringSerializer(), new StringSerializer()); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get()); - - // Cluster metadata will only be updated on calling onSend. - Assert.assertNull(MockProducerInterceptor.CLUSTER_META.get()); - - producer.close(); - Assert.assertEquals(1, MockProducerInterceptor.INIT_COUNT.get()); - Assert.assertEquals(1, MockProducerInterceptor.CLOSE_COUNT.get()); - } finally { - // cleanup since we are using mutable static variables in MockProducerInterceptor - MockProducerInterceptor.resetCounters(); - } - } - - @Test - public void testOsDefaultSocketBufferSizes() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - PubsubProducer producer = new PubsubProducer<>( - config, new ByteArraySerializer(), new ByteArraySerializer()); - producer.close(); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketSendBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Test(expected = KafkaException.class) - public void testInvalidSocketReceiveBufferSize() throws Exception { - Map config = new HashMap<>(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); - new PubsubProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); - } - */ -} +}*/ From edacf4e5478dcdabaa04a49793f60d75785fb4f0 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 1 Mar 2017 09:52:17 -0800 Subject: [PATCH 110/140] Working on unit tests, need to mock the publisher calls --- .../google/pubsub/clients/ProducerThread.java | 5 +- .../clients/producer/PubsubProducer.java | 28 +-------- .../pubsub/common/PubsubChannelUtil.java | 12 ++-- .../clients/producer/PubsubProducerTest.java | 63 ++++++++++++++----- .../pubsub/common/PubsubChannelUtilTest.java | 51 +++++++++++++++ 5 files changed, 110 insertions(+), 49 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 87a6afc6..46e9e20c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -37,8 +37,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); - for (int i = 0; i < 1; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); + for (int i = 0; i < 10; i++) { producer.send( msg, new Callback() { @@ -52,7 +52,6 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { } ); } - Thread.sleep(5000); producer.close(); } catch (InterruptedException e) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 291ba7a9..063093d3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,45 +25,27 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.clients.producer.internals.PubsubFutureRecordMetadata; -import io.grpc.ManagedChannel; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.network.Selectable; -import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketOptions; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -73,11 +55,7 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. @@ -123,7 +101,8 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = channelUtil.createPublisherFutureStub(); + if (keySerializer == null) { this.keySerializer = configs.getConfiguredInstance( @@ -152,7 +131,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); perTopicBatches = Collections.synchronizedMap(new HashMap<>()); - String threadName = "pubsub-producer-network-thread"; log.debug("Producer successfully initialized."); } @@ -305,7 +283,7 @@ public void close() { */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { - throw new IllegalArgumentException("Timout cannot be negative."); + throw new IllegalArgumentException("Timeout cannot be negative."); } channelUtil.closeChannel(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 1991aa38..ac8c7a95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -16,20 +16,19 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import io.grpc.auth.ClientAuthInterceptor; -import io.grpc.internal.ManagedChannelImpl; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +/** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { private static final String ENDPOINT = "pubsub.googleapis.com"; @@ -41,12 +40,17 @@ public class PubsubChannelUtil { private ManagedChannel channel; private CallCredentials callCredentials; + /* Constructs instance with populated credentials and channel */ public PubsubChannelUtil() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } + public PublisherFutureStub createPublisherFutureStub() { + return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); + } + public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index 98f6b889..a8112566 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -14,30 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/*package com.google.kafka.clients.producer; +package com.google.pubsub.clients.producer; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.network.Selectable; +import com.google.common.collect.ImmutableMap; +import java.util.StringTokenizer; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.test.MockMetricsReporter; -import org.apache.kafka.test.MockProducerInterceptor; -import org.apache.kafka.test.MockSerializer; +import org.apache.kafka.common.config.ConfigException; + + import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.modules.junit4.PowerMockRunner; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("javax.management.*") public class PubsubProducerTest { + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + + /* Constructor Tests */ @Test public void testConstructorWithSerializers() { Properties props = new Properties(); @@ -46,11 +47,39 @@ public void testConstructorWithSerializers() { } @Test(expected = ConfigException.class) - public void testNoSerializerProvided() { + public void testConstructorNoSerializerProvided() { Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props); + props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); + new PubsubProducer(props).close(); } + @Test(expected = ConfigException.class) + public void testConstructorNoProjectProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + new PubsubProducer(props).close(); + } + + /* send() tests */ + /* @Test(expected = RuntimeException.class) + public void testSendPublisherClosed() { + + }*/ + + private PubsubProducer getNewProducer() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("project", "dataproc-kafka-test") + .build() + ); + + return new PubsubProducer(props); + } -}*/ +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java new file mode 100644 index 00000000..2c5ad730 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.common; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PubsubChannelUtilTest { + + private static PubsubChannelUtil channelUtil; + + @BeforeClass + public static void setUp() throws IOException { + channelUtil = new PubsubChannelUtil(); + } + + @AfterClass + public static void tearDown() { + channelUtil.closeChannel(); + } + + @Test + public void testGetCallCredentials() throws IOException { + assertNotNull(channelUtil.callCredentials()); + channelUtil.closeChannel(); + } + + @Test + public void testGetChannel() { + assertNotNull(channelUtil.channel()); + } +} From 8fa35669d9d7a299eb8e20ee72c827f2c215b487 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 10:33:32 -0800 Subject: [PATCH 111/140] Continuing work on testing and builder for producer --- .../google/pubsub/clients/ProducerThread.java | 5 +- .../clients/producer/PubsubProducer.java | 100 ++++++++++++++++-- .../pubsub/common/PubsubChannelUtil.java | 4 - .../clients/producer/PubsubProducerTest.java | 35 +----- 4 files changed, 99 insertions(+), 45 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 46e9e20c..4cfa32cf 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -25,7 +25,10 @@ public ProducerThread(String s, Properties props, String topic) throws IOExcepti .isAcks(props.getProperty("acks").matches("1|all")) .build(); this.command = s; - this.producer = new PubsubProducer<>(props); + //this.producer = new PubsubProducer<>(props); + this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) + .batchSize(props.getProperty("batch.size")) + .) this.topic = topic; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 063093d3..0afd92ee 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.producer; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -25,6 +26,8 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import java.io.IOError; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -64,17 +67,31 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private PublisherFutureStub publisher; - private String project; - private Serializer keySerializer; - private Serializer valueSerializer; - private int batchSize; - private boolean isAcks; - private boolean closed = false; - private Map> perTopicBatches; + private final PublisherFutureStub publisher; + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final int batchSize; + private final boolean isAcks; + private final Map> perTopicBatches; private final int maxRequestSize; private final Time time; - private PubsubChannelUtil channelUtil; + private final PubsubChannelUtil channelUtil; + + private boolean closed = false; + + private PubsubProducer(Builder builder) { + publisher = builder.publisher; + project = builder.project; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; + batchSize = builder.batchSize; + isAcks = builder.isAcks; + perTopicBatches = builder.perTopicBatches; + maxRequestSize = builder.maxRequestSize; + time = builder.time; + channelUtil = builder.channelUtil; + } public PubsubProducer(Map configs) { this(new PubsubProducerConfig(configs), null, null); @@ -101,7 +118,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = channelUtil.createPublisherFutureStub(); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -291,6 +308,69 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + public static class Builder { + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private PubsubChannelUtil channelUtil; + private PublisherFutureStub publisher; + private int batchSize; + private boolean isAcks; + private Map> perTopicBatches; + private int maxRequestSize; + private Time time; + + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + this.project = project; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + setDefaults(); + } + + private void setDefaults() { + // this is where to set 'regular' fields w/o side effects + this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; + this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; + this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.time = new SystemTime(); + } + + public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + + public Builder batchSize(int val) { + Preconditions.checkArgument(val > 0); + batchSize = val; + return this; + } + + public Builder isAcks(boolean val) { isAcks = val; return this; } + + public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + + public Builder maxRequestSize(int val) { + Preconditions.checkArgument(val >= 0); + maxRequestSize = val; + return this; + } + + public Builder time(Time val) { time = val; return this; } + + public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + + public PubsubProducer build() throws IOException { + // this is where to set fields w/ side effects + if (channelUtil == null) { + this.channelUtil = new PubsubChannelUtil(); + } + if (publisher == null) { + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + } + return new PubsubProducer(this); + } + } + /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ private static class FutureFailure implements Future { private final ExecutionException exception; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index ac8c7a95..edfe899b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -47,10 +47,6 @@ public PubsubChannelUtil() throws IOException { channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } - public PublisherFutureStub createPublisherFutureStub() { - return PublisherGrpc.newFutureStub(channel).withCallCredentials(callCredentials); - } - public CallCredentials callCredentials() { return callCredentials; } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java index a8112566..32555be2 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -30,45 +30,20 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PubsubProducerTest { private static final String TOPIC = "testTopic"; private static final String MESSAGE = "testMessage"; - /* Constructor Tests */ - @Test - public void testConstructorWithSerializers() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props, new ByteArraySerializer(), new ByteArraySerializer()).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoSerializerProvided() { - Properties props = new Properties(); - props.setProperty(PubsubProducerConfig.PROJECT_CONFIG, "dataproc-kafka-test"); - new PubsubProducer(props).close(); - } - - @Test(expected = ConfigException.class) - public void testConstructorNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - new PubsubProducer(props).close(); - } - /* send() tests */ - /* @Test(expected = RuntimeException.class) + @Test public void testSendPublisherClosed() { + // mock the PublisherFutureStub - }*/ + // mock the PubsubChannelUtil + // construct using testing constructor, every other param normal + } private PubsubProducer getNewProducer() { Properties props = new Properties(); From 45d127451f1abceabf7c19ce71b86ac5dffd5394 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 2 Mar 2017 15:26:06 -0800 Subject: [PATCH 112/140] Builder is implemented. --- .../google/pubsub/clients/ProducerThread.java | 11 +++-------- .../pubsub/clients/ProducerThreadPool.java | 2 +- .../clients/producer/PubsubProducer.java | 19 ++++++++++--------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java index 4cfa32cf..4f3e3427 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java @@ -1,7 +1,7 @@ package com.google.pubsub.clients; -import com.google.api.client.googleapis.MethodOverride.Builder; import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; @@ -24,11 +24,6 @@ public ProducerThread(String s, Properties props, String topic) throws IOExcepti .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .build(); - this.command = s; - //this.producer = new PubsubProducer<>(props); - this.producer = new PubsubProducer(new PubsubProducer.Builder(props.getProperty("project"), StringSerializer.class, StringSerializer.class) - .batchSize(props.getProperty("batch.size")) - .) this.topic = topic; } @@ -40,8 +35,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "message" + command); - for (int i = 0; i < 10; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); + for (int i = 0; i < 1; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java index 5533c79f..36ef8716 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java @@ -16,7 +16,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 0afd92ee..f7d47d24 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -26,7 +27,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOError; import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -83,14 +83,14 @@ public class PubsubProducer implements Producer { private PubsubProducer(Builder builder) { publisher = builder.publisher; project = builder.project; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; batchSize = builder.batchSize; isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; time = builder.time; channelUtil = builder.channelUtil; + keySerializer = builder.keySerializer; + valueSerializer = builder.valueSerializer; } public PubsubProducer(Map configs) { @@ -165,7 +165,7 @@ public Future send(ProducerRecord record) { * Send a record and invoke the given callback when the record has been acknowledged by the server */ public Future send(ProducerRecord record, Callback callback) { - log.info("Received " + record.toString()); + log.trace("Received " + record.toString()); if (closed) { throw new RuntimeException("Publisher is closed"); } @@ -252,7 +252,7 @@ public void onFailure(Throwable t) { private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Messge is " + size + " bytes which is larger than max request size you have" + throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" + " configured"); } } @@ -308,10 +308,10 @@ public void close(long timeout, TimeUnit unit) { closed = true; } - public static class Builder { + public static class Builder { private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; private PubsubChannelUtil channelUtil; private PublisherFutureStub publisher; @@ -321,7 +321,8 @@ public static class Builder { private int maxRequestSize; private Time time; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; From 4dcf7247e8061f46d544bb4bd6c06771baf5b809 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 09:46:08 -0800 Subject: [PATCH 113/140] Minor fixes to producer's classes --- .../pubsub/clients/consumer/PubsubConsumer.java | 3 --- .../pubsub/clients/producer/PubsubProducer.java | 2 +- .../com/google/pubsub/common/PubsubChannelUtil.java | 13 +++++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index e5abef92..285c5d8e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -12,9 +12,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ -<<<<<<< HEAD:mapped-api/clients/src/main/java/com/google/kafka/clients/consumer/PubsubConsumer.java -package com.google.kafka.cients.consumer; -======= package com.google.pubsub.clients.consumer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f7d47d24..e9ca5753 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -360,7 +360,7 @@ public Builder maxRequestSize(int val) { public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } - public PubsubProducer build() throws IOException { + public PubsubProducer build() { // this is where to set fields w/ side effects if (channelUtil == null) { this.channelUtil = new PubsubChannelUtil(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index edfe899b..9088d5bf 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Sets up the pub/sub grpc functionality */ public class PubsubChannelUtil { + private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); private static final String ENDPOINT = "pubsub.googleapis.com"; private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); @@ -41,8 +44,14 @@ public class PubsubChannelUtil { private CallCredentials callCredentials; /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() throws IOException { - GoogleCredentials credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + public PubsubChannelUtil() { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); + } catch (IOException exception) { + log.error("Exception occurred: " + exception.getMessage()); + return; + } callCredentials = MoreCallCredentials.from(credentials); channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); } From 5d8eccaffe3df16cd9c27c000b14a8e815103ca2 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 6 Mar 2017 10:06:42 -0800 Subject: [PATCH 114/140] Omitting unit tests and unused classes --- .../clients/producer/PubsubProducerTest.java | 60 ------------------- .../pubsub/common/PubsubChannelUtilTest.java | 51 ---------------- 2 files changed, 111 deletions(-) delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index 32555be2..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import com.google.common.collect.ImmutableMap; -import java.util.StringTokenizer; -import java.util.concurrent.ExecutionException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.config.ConfigException; - - -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - - /* send() tests */ - @Test - public void testSendPublisherClosed() { - // mock the PublisherFutureStub - - // mock the PubsubChannelUtil - // construct using testing constructor, every other param normal - } - - private PubsubProducer getNewProducer() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "dataproc-kafka-test") - .build() - ); - - return new PubsubProducer(props); - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java deleted file mode 100644 index 2c5ad730..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/common/PubsubChannelUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.pubsub.common; - -import static org.junit.Assert.assertNotNull; - -import java.io.IOException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PubsubChannelUtilTest { - - private static PubsubChannelUtil channelUtil; - - @BeforeClass - public static void setUp() throws IOException { - channelUtil = new PubsubChannelUtil(); - } - - @AfterClass - public static void tearDown() { - channelUtil.closeChannel(); - } - - @Test - public void testGetCallCredentials() throws IOException { - assertNotNull(channelUtil.callCredentials()); - channelUtil.closeChannel(); - } - - @Test - public void testGetChannel() { - assertNotNull(channelUtil.channel()); - } -} From 66ecada22001f6669ccdcd97b1e324ee3c963dd8 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 11:06:38 -0800 Subject: [PATCH 115/140] Eliminated deprecated client use in pom; added to .travis.yml --- .travis.yml | 1 + pubsub-mapped-api/pom.xml | 27 +------------------ .../clients/producer/PubsubProducer.java | 2 -- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70f2177d..fb2a5a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: script: - mvn -q -B -f jms-light/pom.xml clean verify - mvn -q -B -f kafka-connector/pom.xml clean verify + - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 - mvn -q -B -f load-test-framework/pom.xml clean verify diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 60f4ca73..3b892924 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -8,15 +8,10 @@ MappedApi http://maven.apache.org - - com.google.pubsub - cloud-pubsub-client - 0.2-EXPERIMENTAL - com.google.cloud google-cloud-pubsub - 0.9.2-alpha + 0.9.4-alpha junit @@ -63,26 +58,6 @@ log4j 1.2.17 - - io.grpc - grpc-all - 1.0.1 - - - io.grpc - grpc-netty - 1.0.1 - - - io.grpc - grpc-protobuf - 1.0.1 - - - io.grpc - grpc-stub - 1.0.1 - org.mockito mockito-all diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index e9ca5753..83550495 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -16,7 +16,6 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -27,7 +26,6 @@ import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; -import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; From 03129b4d1b90115722e84826a3a9c60d2a6942a9 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Wed, 8 Mar 2017 11:59:54 -0800 Subject: [PATCH 116/140] Modified .travis.yml to run mapped_api with jdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fb2a5a34..8a4957b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: script: - mvn -q -B -f jms-light/pom.xml clean verify - mvn -q -B -f kafka-connector/pom.xml clean verify - - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 + - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - mvn -q -B -f load-test-framework/pom.xml clean verify From ccaf103ea4b5b33aec5f5a123078ce8c7a93e781 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 9 Mar 2017 15:39:22 -0800 Subject: [PATCH 117/140] Fixed requested changes on style, naming, and licenses. --- .../clients/consumer/PubsubConsumer.java | 429 +----------------- .../clients/producer/PubsubProducer.java | 159 ++++--- .../producer/PubsubProducerConfig.java | 59 ++- .../clients/{ => tests}/ProducerThread.java | 30 +- .../{ => tests}/ProducerThreadPool.java | 48 +- .../pubsub/common/PubsubChannelUtil.java | 29 +- 6 files changed, 209 insertions(+), 545 deletions(-) rename pubsub-mapped-api/src/main/java/com/google/pubsub/clients/{ => tests}/ProducerThread.java (66%) rename pubsub-mapped-api/src/main/java/com/google/pubsub/clients/{ => tests}/ProducerThreadPool.java (54%) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 285c5d8e..5e8fd2d8 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -1,69 +1,39 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.clients.consumer; import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.common.Cluster; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.metrics.JmxReporter; -import org.apache.kafka.common.metrics.MetricConfig; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.MetricsReporter; -import org.apache.kafka.common.network.ChannelBuilder; -import org.apache.kafka.common.network.Selector; -import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetSocketAddress; import java.util.Collection; -import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; public class PubsubConsumer implements Consumer { @@ -88,502 +58,133 @@ public PubsubConsumer(Properties properties, } - /** - * Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning - * partitions using {@link #assign(Collection)} then this will simply return the same partitions that - * were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned - * to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the - * process of getting reassigned). - * @return The set of partitions currently assigned to this consumer - */ public Set assignment() { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the current subscription. Will return the same topics used in the most recent call to - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made. - * @return The set of topics currently subscribed to - */ public Set subscription() { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to the given list of topics to get dynamically - * assigned partitions. Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). Note that it is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that belong to a particular - * group and will trigger a rebalance operation if one of the following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- *

- * When any of these events are triggered, the provided listener will be invoked first to indicate that - * the consumer's assignment has been revoked, and then again when the new assignment has been received. - * Note that this listener will immediately override any listener set in a previous call to subscribe. - * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics - * subscribed in this call. See {@link ConsumerRebalanceListener} for more details. - * - * @param topics The list of topics to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ public void subscribe(Collection topics, ConsumerRebalanceListener listener) { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which - * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer - * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets - * to be reset. You should also provide your own listener if you are doing your own offset - * management since the listener gives you an opportunity to commit offsets before a rebalance finishes. - * - * @param topics The list of topics to subscribe to - * @throws IllegalArgumentException If topics is null or contains null or empty elements - */ @Override public void subscribe(Collection topics) { throw new NotImplementedException("Not yet implemented"); } - /** - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics - * existing at the time of check. - * - *

- * As part of group management, the consumer will keep track of the list of consumers that - * belong to a particular group and will trigger a rebalance operation if one of the - * following events trigger - - *

    - *
  • Number of partitions change for any of the subscribed list of topics - *
  • Topic is created or deleted - *
  • An existing member of the consumer group dies - *
  • A new member is added to an existing consumer group via the join API - *
- * - * @param pattern Pattern to subscribe to - * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the - * subscribed topics - * @throws IllegalArgumentException If pattern is null - */ @Override public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { throw new NotImplementedException("Not yet implemented"); } - /** - * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)}. This - * also clears any partitions directly assigned through {@link #assign(Collection)}. - */ public void unsubscribe() { throw new NotImplementedException("Not yet implemented"); } - /** - * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment - * and will replace the previous assignment (if there is one). - * - * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * Manual topic assignment through this method does not use the consumer's group management - * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic - * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)} - * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}. - * - * @param partitions The list of partitions to assign this consumer - * @throws IllegalArgumentException If partitions is null or contains null or empty topics - */ @Override public void assign(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - - /** - * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have - * subscribed to any topics or partitions before polling for data. - *

- * On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last - * consumed offset can be manually set through {@link #seek(TopicPartition, long)} or automatically set as the last committed - * offset for the subscribed list of partitions - * - * - * @param timeout The time, in milliseconds, spent waiting in poll if data is not available in the buffer. - * If 0, returns immediately with any records that are available currently in the buffer, else returns empty. - * Must not be negative. - * @return map of topic to records since the last fetch for the subscribed list of topics and partitions - * - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if the offset for a partition or set of - * partitions is undefined or out of range and no offset reset policy has been configured - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed - * topics or to the configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) - * @throws java.lang.IllegalArgumentException if the timeout value is negative - * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any - * partitions to consume from - */ @Override public ConsumerRecords poll(long timeout) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ @Override public void commitSync() { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit the specified offsets for the specified list of topics and partitions. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is - * encountered (in which case it is thrown to the caller). - * - * @param offsets A map of offsets by partition with associated metadata - * @throws org.apache.kafka.clients.consumer.CommitFailedException if the commit failed and cannot be retried. - * This can only occur if you are using automatic group management with {@link #subscribe(Collection)}, - * or if there is an active group with the same groupId which is using group management. - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. if offset metadata - * is too large or if the committed offset is invalid). - */ @Override public void commitSync(final Map offsets) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for all the subscribed list of topics and partition. - * Same as {@link #commitAsync(OffsetCommitCallback) commitAsync(null)} - */ @Override public void commitAsync() { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit offsets returned on the last {@link #poll(long) poll()} for the subscribed list of topics and partitions. - *

- * This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after - * every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param callback Callback to invoke when the commit completes - */ @Override public void commitAsync(OffsetCommitCallback callback) { throw new NotImplementedException("Not yet implemented"); } - /** - * Commit the specified offsets for the specified list of topics and partitions to Kafka. - *

- * This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every - * rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API - * should not be used. The committed offset should be the next message your application will consume, - * i.e. lastProcessedMessageOffset + 1. - *

- * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback - * (if provided) or discarded. - * - * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it - * is safe to mutate the map after returning. - * @param callback Callback to invoke when the commit completes - */ @Override public void commitAsync(final Map offsets, OffsetCommitCallback callback) { throw new NotImplementedException("Not yet implemented"); } - /** - * Overrides the fetch offsets that the consumer will use on the next {@link #poll(long) poll(timeout)}. If this API - * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that - * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets - */ @Override public void seek(TopicPartition partition, long offset) { throw new NotImplementedException("Not yet implemented"); } - /** - * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the - * first offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the first offset for all of the currently assigned partitions. - */ public void seekToBeginning(Collection partitions) { } - /** - * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the - * final offset in all partitions only when {@link #poll(long)} or {@link #position(TopicPartition)} are called. - * If no partition is provided, seek to the final offset for all of the currently assigned partitions. - */ public void seekToEnd(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the offset of the next record that will be fetched (if a record with that offset exists). - * - * @param partition The partition to get the position for - * @return The offset - * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for - * the partition - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public long position(TopicPartition partition) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the last committed offset for the given partition (whether the commit happened by this process or - * another). This offset will be used as the position for the consumer in the event of a failure. - *

- * This call may block to do a remote call if the partition in question isn't assigned to this consumer or if the - * consumer hasn't yet initialized its cache of committed offsets. - * - * @param partition The partition to check - * @return The last committed offset and metadata or null if there was no prior commit - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the - * configured groupId - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public OffsetAndMetadata committed(TopicPartition partition) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the metrics kept by the consumer - */ public Map metrics() { throw new NotImplementedException("Not yet implemented"); } - /** - * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it - * does not already have any metadata about the given topic. - * - * @param topic The topic to get partition metadata for - * @return The list of partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public List partitionsFor(String topic) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get metadata about partitions for all topics that the user is authorized to view. This method will issue a - * remote call to the server. - * @return The map of topics and its partitions - * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this - * function is called - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while - * this function is called - * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before - * expiration of the configured request timeout - * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors - */ public Map> listTopics() { throw new NotImplementedException("Not yet implemented"); } - /** - * Suspend fetching from the requested partitions. Future calls to {@link #poll(long)} will not return - * any records from these partitions until they have been resumed using {@link #resume(Collection)}. - * Note that this method does not affect partition subscription. In particular, it does not cause a group - * rebalance when automatic assignment is used. - * @param partitions The partitions which should be paused - */ public void pause(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Resume specified partitions which have been paused with {@link #pause(Collection)}. New calls to - * {@link #poll(long)} will return records from these partitions if there are any to be fetched. - * If the partitions were not previously paused, this method is a no-op. - * @param partitions The partitions which should be resumed - */ public void resume(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the set of partitions that were previously paused by a call to {@link #pause(Collection)}. - * - * @return The set of paused partitions - */ public Set paused() { throw new NotImplementedException("Not yet implemented"); } - /** - * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. - * - * This is a blocking call. The consumer does not have to be assigned the partitions. - * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null - * will be returned for that partition. - * - * Notice that this method may block indefinitely if the partition does not exist. - * - * @param timestampsToSearch the mapping from partition to the timestamp to look up. - * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater - * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no - * such message. - * @throws IllegalArgumentException if the target timestamp is negative. - * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up - * the offsets by timestamp. - */ public Map offsetsForTimes(Map timestampsToSearch) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the first offset for the given partitions. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToBeginning(Collection) - * - * @param partitions the partitions to get the earliest offsets. - * @return The earliest available offsets for the given partitions - */ public Map beginningOffsets(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming - * message, i.e. the offset of the last available message + 1. - *

- * Notice that this method may block indefinitely if the partition does not exist. - * This method does not change the current consumer position of the partitions. - * - * @see #seekToEnd(Collection) - * - * @param partitions the partitions to get the end offsets. - * @return The end offsets for the given partitions. - */ public Map endOffsets(Collection partitions) { throw new NotImplementedException("Not yet implemented"); } - /** - * Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. - * If auto-commit is enabled, this will commit the current offsets if possible within the default - * timeout. See {@link #close(long, TimeUnit)} for details. Note that {@link #wakeup()} - * cannot be used to interrupt close. - * - * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted - * before or while this function is called - */ public void close() { } - /** - * Tries to close the consumer cleanly within the specified timeout. This method waits up to - * timeout for the consumer to complete pending commits and leave the group. - * If auto-commit is enabled, this will commit the current offsets if possible within the - * timeout. If the consumer is unable to complete offset commits and gracefully leave the group - * before the timeout expires, the consumer is force closed. Note that {@link #wakeup()} cannot be - * used to interrupt close. - * - * @param timeout The maximum time to wait for consumer to close gracefully. The value must be - * non-negative. Specifying a timeout of zero means do not wait for pending requests to complete. - * @param timeUnit The time unit for the timeout - * @throws InterruptException If the thread is interrupted before or while this function is called - * @throws IllegalArgumentException If the timeout is negative. - */ public void close(long timeout, TimeUnit timeUnit) { throw new NotImplementedException("Not yet implemented"); } - /** - * Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. - * The thread which is blocking in an operation will throw {@link org.apache.kafka.common.errors.WakeupException}. - * If no thread is blocking in a method which can throw {@link org.apache.kafka.common.errors.WakeupException}, the next call to such a method will raise it instead. - */ public void wakeup() { throw new NotImplementedException("Not yet implemented"); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 83550495..1a573612 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -1,17 +1,18 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.clients.producer; @@ -97,7 +98,8 @@ public PubsubProducer(Map configs) { public PubsubProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), + this(new PubsubProducerConfig( + PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); } @@ -107,16 +109,19 @@ public PubsubProducer(Properties properties) { public PubsubProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new PubsubProducerConfig(PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), + this(new PubsubProducerConfig( + PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); } - private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, Serializer valueSerializer) { + private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, + Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) + .withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -150,17 +155,14 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } /** - * Send the given record asynchronously and return a future which will eventually contain the response information. - * - * @param record The record to send - * @return A future which will eventually contain the response information + * Sends the given record. */ public Future send(ProducerRecord record) { return send(record, null); } /** - * Send a record and invoke the given callback when the record has been acknowledged by the server + * Sends the given record and invokes the specified callback. */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); @@ -174,7 +176,8 @@ public Future send(ProducerRecord record, Callback callbac byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes.put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + attributes + .put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -190,9 +193,9 @@ public Future send(ProducerRecord record, Callback callbac PubsubMessage message = PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(valueBytes)) - .putAllAttributes(attributes) - .build(); + .setData(ByteString.copyFrom(valueBytes)) + .putAllAttributes(attributes) + .build(); List batch = perTopicBatches.get(topic); if (batch == null) { batch = new ArrayList<>(batchSize); @@ -203,20 +206,22 @@ public Future send(ProducerRecord record, Callback callbac long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); RecordAccumulator.RecordAppendResult result = new RecordAppendResult( new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, false); + timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, + false); if (result.batchIsFull) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(batch) + .build(); doSend(request, callback, result); } return result.future; } - private Future doSend(PublishRequest request, Callback callback, RecordAppendResult result) { + private Future doSend(PublishRequest request, Callback callback, + RecordAppendResult result) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -243,58 +248,58 @@ public void onFailure(Throwable t) { perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { - return new FutureFailure(e); + log.error("Exception occurred during send: " + e); + return null; } return result.future; } private void checkRecordSize(int size) { if (size > this.maxRequestSize) { - throw new RecordTooLargeException("Message is " + size + " bytes which is larger than max request size you have" - + " configured"); + throw new RecordTooLargeException( + "Message is " + size + " bytes which is larger than max request size you have" + + " configured"); } } /** - * Flush any accumulated records from the producer. Blocks until all sends are complete. + * Flushes records that have accumulated. */ public void flush() { log.debug("Flushing..."); for (String topic : perTopicBatches.keySet()) { PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); + .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .addAllMessages(perTopicBatches.get(topic)) + .build(); doSend(request, null, null); } } /** - * Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change - * over time so this list should not be cached. + * Not supported by this implementation. */ public List partitionsFor(String topic) { throw new NotImplementedException("Partitions not supported"); } /** - * Return a map of metrics maintained by the producer + * Not supported by this implementation. */ public Map metrics() { throw new NotImplementedException("Metrics not supported."); } /** - * Close this producer + * Closes the producer. */ public void close() { close(0, null); } /** - * Tries to close the producer cleanly within the specified timeout. If the close does not complete within the - * timeout, fail any pending send requests and force close the producer. + * Closes the producer with the given timeout. */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { @@ -306,7 +311,12 @@ public void close(long timeout, TimeUnit unit) { closed = true; } + /** + * PubsubProducer.Builder is used to create an instance of the publisher, with the specified + * properties and configurations. + */ public static class Builder { + private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; @@ -320,7 +330,8 @@ public static class Builder { private Time time; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { - Preconditions.checkArgument(project != null && keySerializer != null && valueSerializer != null); + Preconditions + .checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; @@ -336,7 +347,10 @@ private void setDefaults() { this.time = new SystemTime(); } - public Builder publisherFutureStub(PublisherFutureStub val) { publisher = val; return this; } + public Builder publisherFutureStub(PublisherFutureStub val) { + publisher = val; + return this; + } public Builder batchSize(int val) { Preconditions.checkArgument(val > 0); @@ -344,9 +358,15 @@ public Builder batchSize(int val) { return this; } - public Builder isAcks(boolean val) { isAcks = val; return this; } + public Builder isAcks(boolean val) { + isAcks = val; + return this; + } - public Builder perTopicBatches(Map> val) { perTopicBatches = val; return this; } + public Builder perTopicBatches(Map> val) { + perTopicBatches = val; + return this; + } public Builder maxRequestSize(int val) { Preconditions.checkArgument(val >= 0); @@ -354,9 +374,15 @@ public Builder maxRequestSize(int val) { return this; } - public Builder time(Time val) { time = val; return this; } + public Builder time(Time val) { + time = val; + return this; + } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; } + public Builder pubsubChannelUtil(PubsubChannelUtil val) { + channelUtil = val; + return this; + } public PubsubProducer build() { // this is where to set fields w/ side effects @@ -364,38 +390,11 @@ public PubsubProducer build() { this.channelUtil = new PubsubChannelUtil(); } if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()).withCallCredentials(channelUtil.callCredentials()); + this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) + .withCallCredentials(channelUtil.callCredentials()); } return new PubsubProducer(this); } } - /** Taken from KafkaProducer.java since FutureFailure is private inside that class. */ - private static class FutureFailure implements Future { - private final ExecutionException exception; - - public FutureFailure(Exception e) { - this.exception = new ExecutionException(e); - } - - public boolean cancel(boolean interrupt) { - return false; - } - - public RecordMetadata get() throws ExecutionException { - throw this.exception; - } - - public RecordMetadata get(long timeout, TimeUnit unit) throws ExecutionException { - throw this.exception; - } - - public boolean isCancelled() { - return false; - } - - public boolean isDone() { - return true; - } - } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index fc20f034..cbaed929 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -1,17 +1,19 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + package com.google.pubsub.clients.producer; import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; @@ -25,20 +27,26 @@ import org.apache.kafka.common.config.ConfigDef.Type; import org.apache.kafka.common.serialization.Serializer; +/** + * Provides the configurations for a PubsubProducer instance. + */ public class PubsubProducerConfig extends AbstractConfig { private static final ConfigDef CONFIG; public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; - private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements the Serializer interface."; + private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements " + + "the Serializer interface."; public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; - private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that implements the Serializer interface."; + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that " + + "implements the Serializer interface."; public static final String BATCH_SIZE_CONFIG = "batch.size"; private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; public static final String ACKS_CONFIG = "acks"; - private static final String ACKS_DOC = "Whether server acks are needed before a publish request completes."; + private static final String ACKS_DOC = "Whether server acks are needed before a publish " + + "request completes."; public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; @@ -56,18 +64,22 @@ public class PubsubProducerConfig extends AbstractConfig { .define( KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define( - VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, + VALUE_SERIALIZER_CLASS_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) - .define(ACKS_CONFIG, Type.STRING, DEFAULT_ACKS, Importance.MEDIUM, ACKS_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); + .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, + BATCH_SIZE_DOC) + .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) + .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), + Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); } PubsubProducerConfig(Map properties) { super(CONFIG, properties); } - public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { + public static Map addSerializerToConfig(Map configs, + Serializer keySerializer, Serializer valueSerializer) { Map newConfigs = new HashMap(); newConfigs.putAll(configs); if (keySerializer != null) { @@ -79,7 +91,8 @@ public static Map addSerializerToConfig(Map conf return newConfigs; } - public static Properties addSerializerToConfig(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + public static Properties addSerializerToConfig(Properties properties, + Serializer keySerializer, Serializer valueSerializer) { Properties newProperties = new Properties(); newProperties.putAll(properties); if (keySerializer != null) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java similarity index 66% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index 4f3e3427..fc02a198 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -1,8 +1,23 @@ -package com.google.pubsub.clients; +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.tests; import com.google.pubsub.clients.producer.PubsubProducer; import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import java.io.IOException; import java.util.Properties; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; @@ -11,20 +26,25 @@ import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; - +/** + * Creates a thread that sends a given series of messages. Serves as a sanity check for the + * PubsubProducer implementation. + */ public class ProducerThread implements Runnable { private String command; private PubsubProducer producer; private String topic; private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); + private int numMessages; - public ProducerThread(String s, Properties props, String topic) throws IOException { + public ProducerThread(String s, Properties props, String topic, int numMessages) { this.command = s; this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .build(); this.topic = topic; + this.numMessages = numMessages; } public void run() { @@ -36,7 +56,7 @@ public void run() { private void processCommand() { try { ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); - for (int i = 0; i < 1; i++) { + for (int i = 0; i < numMessages; i++) { producer.send( msg, new Callback() { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java similarity index 54% rename from pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java rename to pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 36ef8716..820f889f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -1,22 +1,43 @@ -package com.google.pubsub.clients; +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// -import java.io.IOException; +package com.google.pubsub.clients.tests; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.pubsub.clients.producer.PubsubProducer; import java.util.Properties; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; +/** + * Example main class to start up several ProducerThreads with a specified topic. + * Number of threads will also be specified by a command-line argument. + * arg[0] = topic, arg[1] = number of threads, arg[2] = number of messages to send. + * Topic is required; if number of threads is not specified, 2 is the default. + * If number of messages is not specified, 1 is the default. + */ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - public static void main(String[] args) throws IOException { + public static void main(String[] args) { + Preconditions.checkArgument(args[0] != null); ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -38,8 +59,17 @@ public void uncaughtException(Thread t, Throwable e) { .build() ); - for (int i = 0; i < 1; i++) { - Runnable worker = new ProducerThread("" + i, props, args[0]); + int numThreads = 2; + if (args[1] != null) { + numThreads = Integer.parseInt(args[1]); + } + int numMessages = 1; + if (args[2] != null) { + numMessages = Integer.parseInt(args[2]); + } + + for (int i = 0; i < numThreads; i++) { + Runnable worker = new ProducerThread("" + i, props, args[0], numMessages); executor.execute(worker); } @@ -56,4 +86,4 @@ public void uncaughtException(Thread t, Throwable e) { Thread.currentThread().interrupt(); } } -} \ No newline at end of file +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 9088d5bf..5bc9f9b1 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -1,17 +1,18 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.common; From 8a1af22715e655b499c876a884b5d7bd7bf8ac09 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 10 Mar 2017 10:57:32 -0800 Subject: [PATCH 118/140] Small fixes to concurrency issues in Producer. --- .../clients/producer/PubsubProducer.java | 49 ++++++------------- .../clients/tests/ProducerThreadPool.java | 6 +-- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 1a573612..daa8606b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PublishRequest; import com.google.pubsub.v1.PublishResponse; @@ -32,10 +33,6 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; -import org.apache.kafka.clients.producer.internals.ProduceRequestResult; -import org.apache.kafka.clients.producer.internals.RecordAccumulator; -import org.apache.kafka.clients.producer.internals.RecordAccumulator.RecordAppendResult; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; @@ -43,8 +40,6 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.SystemTime; -import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,8 +69,8 @@ public class PubsubProducer implements Producer { private final boolean isAcks; private final Map> perTopicBatches; private final int maxRequestSize; - private final Time time; private final PubsubChannelUtil channelUtil; + private final SettableFuture future; private boolean closed = false; @@ -86,10 +81,10 @@ private PubsubProducer(Builder builder) { isAcks = builder.isAcks; perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; - time = builder.time; channelUtil = builder.channelUtil; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; + future = SettableFuture.create(); } public PubsubProducer(Map configs) { @@ -118,7 +113,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - this.time = new SystemTime(); channelUtil = new PubsubChannelUtil(); publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) .withCallCredentials(channelUtil.callCredentials()); @@ -144,12 +138,12 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - + future = SettableFuture.create(); batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = Collections.synchronizedMap(new HashMap<>()); + perTopicBatches = new HashMap<>(); log.debug("Producer successfully initialized."); } @@ -177,7 +171,7 @@ public Future send(ProducerRecord record, Callback callbac if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); attributes - .put(channelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } if (project == null) { @@ -203,25 +197,19 @@ public Future send(ProducerRecord record, Callback callbac } batch.add(message); - long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); - RecordAccumulator.RecordAppendResult result = new RecordAppendResult( - new FutureRecordMetadata(new ProduceRequestResult(), 0, - timestamp, 0, serializedKey.length, valueBytes.length), batch.size() == batchSize, - false); - if (result.batchIsFull) { + if (batch.size() == batchSize) { log.trace("Sending a batch of messages."); PublishRequest request = PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) + .setTopic(String.format(PubsubChannelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(batch) .build(); - doSend(request, callback, result); + doSend(request, callback); } - return result.future; + return future; } - private Future doSend(PublishRequest request, Callback callback, - RecordAppendResult result) { + private Future doSend(PublishRequest request, Callback callback) { try { ListenableFuture response = publisher.publish(request); if (callback != null) { @@ -248,10 +236,10 @@ public void onFailure(Throwable t) { perTopicBatches.clear(); } } catch (InterruptedException | ExecutionException e) { - log.error("Exception occurred during send: " + e); - return null; + future.setException(e); + // TODO: fix this so it's not unused } - return result.future; + return future; } private void checkRecordSize(int size) { @@ -273,7 +261,7 @@ public void flush() { .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) .addAllMessages(perTopicBatches.get(topic)) .build(); - doSend(request, null, null); + doSend(request, null); } } @@ -327,7 +315,6 @@ public static class Builder { private boolean isAcks; private Map> perTopicBatches; private int maxRequestSize; - private Time time; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { Preconditions @@ -344,7 +331,6 @@ private void setDefaults() { this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; - this.time = new SystemTime(); } public Builder publisherFutureStub(PublisherFutureStub val) { @@ -374,11 +360,6 @@ public Builder maxRequestSize(int val) { return this; } - public Builder time(Time val) { - time = val; - return this; - } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { channelUtil = val; return this; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 820f889f..983e0118 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -37,7 +37,7 @@ public class ProducerThreadPool { private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); public static void main(String[] args) { - Preconditions.checkArgument(args[0] != null); + Preconditions.checkArgument(args.length > 0); ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -60,11 +60,11 @@ public void uncaughtException(Thread t, Throwable e) { ); int numThreads = 2; - if (args[1] != null) { + if (args.length > 1) { numThreads = Integer.parseInt(args[1]); } int numMessages = 1; - if (args[2] != null) { + if (args.length > 2) { numMessages = Integer.parseInt(args[2]); } From 246d62b25e0a76999a1375b3dae4617685d43854 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Mon, 13 Mar 2017 16:49:53 -0700 Subject: [PATCH 119/140] Implementing producer using CPS Publisher API. --- .../clients/producer/PubsubProducer.java | 174 +++++++----------- .../clients/tests/ProducerThreadPool.java | 2 +- .../pubsub/common/PubsubChannelUtil.java | 2 - 3 files changed, 68 insertions(+), 110 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index daa8606b..3b4c4961 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -17,17 +17,21 @@ package com.google.pubsub.clients.producer; import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.api.gax.core.RpcFuture; +import com.google.api.gax.core.RpcFutureCallback; +import com.google.api.gax.grpc.BundlingSettings; +import com.google.api.gax.grpc.FlowControlSettings; +import com.google.cloud.pubsub.spi.v1.Publisher; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -40,12 +44,11 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; import org.apache.kafka.common.serialization.Serializer; +import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,30 +64,24 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private final PublisherFutureStub publisher; private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; private final int batchSize; private final boolean isAcks; - private final Map> perTopicBatches; private final int maxRequestSize; - private final PubsubChannelUtil channelUtil; - private final SettableFuture future; + private final Map publishers; private boolean closed = false; private PubsubProducer(Builder builder) { - publisher = builder.publisher; project = builder.project; batchSize = builder.batchSize; isAcks = builder.isAcks; - perTopicBatches = builder.perTopicBatches; maxRequestSize = builder.maxRequestSize; - channelUtil = builder.channelUtil; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; - future = SettableFuture.create(); + publishers = new HashMap<>(); } public PubsubProducer(Map configs) { @@ -113,9 +110,6 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer Serializer valueSerializer) { try { log.trace("Starting the Pubsub producer"); - channelUtil = new PubsubChannelUtil(); - publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) - .withCallCredentials(channelUtil.callCredentials()); if (keySerializer == null) { this.keySerializer = @@ -138,12 +132,11 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - future = SettableFuture.create(); batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - perTopicBatches = new HashMap<>(); + publishers = new HashMap<>(); log.debug("Producer successfully initialized."); } @@ -164,81 +157,71 @@ public Future send(ProducerRecord record, Callback callbac throw new RuntimeException("Publisher is closed"); } - String topic = record.topic(); + TopicName topic = TopicName.create(project, record.topic()); Map attributes = new HashMap<>(); byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic, record.key()); + serializedKey = this.keySerializer.serialize(topic.getTopic(), record.key()); attributes .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } - if (project == null) { - throw new RuntimeException("No project specified."); - } - byte[] valueBytes = ByteString.EMPTY.toByteArray(); if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic, record.value()); + valueBytes = valueSerializer.serialize(topic.getTopic(), record.value()); } checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); + synchronized (publishers) { + if (!publishers.containsKey(topic)) { + try { + Publisher newPub = Publisher.newBuilder(topic) + .setBundlingSettings(BundlingSettings.newBuilder() + .setIsEnabled(true) + .setElementCountThreshold((long) batchSize) + .setDelayThreshold(new Duration(1)) + .setRequestByteThreshold(1000L) + .build()) + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(maxRequestSize) + .build()) + .build(); + publishers.put(topic, newPub); + } catch (IOException e) { + log.error("Exception occurred: " + e); + } + } + } + PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - List batch = perTopicBatches.get(topic); - if (batch == null) { - batch = new ArrayList<>(batchSize); - perTopicBatches.put(topic, batch); - } - batch.add(message); - - if (batch.size() == batchSize) { - log.trace("Sending a batch of messages."); - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(PubsubChannelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(batch) - .build(); - doSend(request, callback); - } - return future; - } - private Future doSend(PublishRequest request, Callback callback) { - try { - ListenableFuture response = publisher.publish(request); - if (callback != null) { - if (isAcks) { - Futures.addCallback( - response, - new FutureCallback() { - public void onSuccess(PublishResponse response) { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } - - public void onFailure(Throwable t) { - callback.onCompletion(null, new Exception(t)); - } - } - ); - } else { - perTopicBatches.clear(); - callback.onCompletion(null, null); - } + RpcFuture messageIdFuture = publishers.get(topic).publish(message); + Future future = SettableFuture.create(); + + if (callback != null) { + if (isAcks) { + messageIdFuture.addCallback(new RpcFutureCallback() { + @Override + public void onFailure(Throwable t) { + callback.onCompletion(null, new ExecutionException(t)); + } + + @Override + public void onSuccess(String result) { + callback.onCompletion(null, null); + } + }); } else { - response.get(); - perTopicBatches.clear(); + callback.onCompletion(null, null); } - } catch (InterruptedException | ExecutionException e) { - future.setException(e); - // TODO: fix this so it's not unused } + return future; } @@ -255,13 +238,12 @@ private void checkRecordSize(int size) { */ public void flush() { log.debug("Flushing..."); - for (String topic : perTopicBatches.keySet()) { - PublishRequest request = - PublishRequest.newBuilder() - .setTopic(String.format(channelUtil.CPS_TOPIC_FORMAT, project, topic)) - .addAllMessages(perTopicBatches.get(topic)) - .build(); - doSend(request, null); + for (TopicName topic : publishers.keySet()) { + try { + publishers.get(topic).shutdown(); + } catch (Exception e) { + log.error("Exception occurred during flush: " + e); + } } } @@ -293,8 +275,13 @@ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { throw new IllegalArgumentException("Timeout cannot be negative."); } - - channelUtil.closeChannel(); + for (TopicName topic : publishers.keySet()) { + try { + publishers.get(topic).shutdown(); + } catch (Exception e) { + log.error("Exception occurred during close: " + e); + } + } log.debug("Closed producer"); closed = true; } @@ -309,11 +296,8 @@ public static class Builder { private final Serializer keySerializer; private final Serializer valueSerializer; - private PubsubChannelUtil channelUtil; - private PublisherFutureStub publisher; private int batchSize; private boolean isAcks; - private Map> perTopicBatches; private int maxRequestSize; public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { @@ -329,15 +313,9 @@ private void setDefaults() { // this is where to set 'regular' fields w/o side effects this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; - this.perTopicBatches = Collections.synchronizedMap(new HashMap<>()); this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; } - public Builder publisherFutureStub(PublisherFutureStub val) { - publisher = val; - return this; - } - public Builder batchSize(int val) { Preconditions.checkArgument(val > 0); batchSize = val; @@ -349,31 +327,13 @@ public Builder isAcks(boolean val) { return this; } - public Builder perTopicBatches(Map> val) { - perTopicBatches = val; - return this; - } - public Builder maxRequestSize(int val) { Preconditions.checkArgument(val >= 0); maxRequestSize = val; return this; } - public Builder pubsubChannelUtil(PubsubChannelUtil val) { - channelUtil = val; - return this; - } - public PubsubProducer build() { - // this is where to set fields w/ side effects - if (channelUtil == null) { - this.channelUtil = new PubsubChannelUtil(); - } - if (publisher == null) { - this.publisher = PublisherGrpc.newFutureStub(channelUtil.channel()) - .withCallCredentials(channelUtil.callCredentials()); - } return new PubsubProducer(this); } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 983e0118..94478810 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -54,7 +54,7 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", "1") + .put("batch.size", "5") .put("linger.ms", "1") .build() ); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java index 5bc9f9b1..062f4d57 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java @@ -17,8 +17,6 @@ package com.google.pubsub.common; import com.google.auth.oauth2.GoogleCredentials; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import io.grpc.auth.MoreCallCredentials; import io.grpc.CallCredentials; import io.grpc.Channel; From 9ad754c0848a0cdd5e6cefcc6c711aac3c8c569d Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 10:10:58 -0700 Subject: [PATCH 120/140] Ensuring that the configs are consistent across types of publisher. --- .../java/com/google/pubsub/clients/mapped/CPSPublisherTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index e100f069..d8fa81b8 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -35,7 +35,7 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) - .batchSize(this.batchSize) + .batchSize(9500000) .isAcks(true) .build(); } From 0626a6ce64e9f3c8336cc13271ff6053c94e15a9 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 11:29:38 -0700 Subject: [PATCH 121/140] Added more configs; added metadata to return for send() --- .../clients/producer/PubsubProducer.java | 52 +++++++++++++------ .../producer/PubsubProducerConfig.java | 14 ++++- .../pubsub/clients/tests/ProducerThread.java | 4 +- .../clients/tests/ProducerThreadPool.java | 2 +- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 3b4c4961..8b977abb 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -22,10 +22,6 @@ import com.google.api.gax.grpc.BundlingSettings; import com.google.api.gax.grpc.FlowControlSettings; import com.google.cloud.pubsub.spi.v1.Publisher; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; @@ -40,6 +36,7 @@ import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.Records; @@ -63,13 +60,16 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); + private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 1000; private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; - private final int batchSize; + private final long batchSize; + private final int bufferMemory; private final boolean isAcks; private final int maxRequestSize; + private final long lingerMs; private final Map publishers; private boolean closed = false; @@ -81,6 +81,8 @@ private PubsubProducer(Builder builder) { maxRequestSize = builder.maxRequestSize; keySerializer = builder.keySerializer; valueSerializer = builder.valueSerializer; + lingerMs = builder.lingerMs; + bufferMemory = builder.bufferMemory; publishers = new HashMap<>(); } @@ -132,10 +134,12 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer } catch (Exception e) { throw new RuntimeException(e); } - batchSize = configs.getInt(PubsubProducerConfig.BATCH_SIZE_CONFIG); + batchSize = configs.getLong(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); + lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); + bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); log.debug("Producer successfully initialized."); @@ -180,12 +184,12 @@ public Future send(ProducerRecord record, Callback callbac Publisher newPub = Publisher.newBuilder(topic) .setBundlingSettings(BundlingSettings.newBuilder() .setIsEnabled(true) - .setElementCountThreshold((long) batchSize) - .setDelayThreshold(new Duration(1)) - .setRequestByteThreshold(1000L) + .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) + .setDelayThreshold(Duration.millis(lingerMs)) + .setRequestByteThreshold(batchSize) .build()) .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(maxRequestSize) + .setMaxOutstandingRequestBytes(bufferMemory) .build()) .build(); publishers.put(topic, newPub); @@ -203,22 +207,24 @@ public Future send(ProducerRecord record, Callback callbac RpcFuture messageIdFuture = publishers.get(topic).publish(message); Future future = SettableFuture.create(); + RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic.getTopic(), 0), + 0, 0, System.currentTimeMillis(), 0, serializedKey.length, valueBytes.length); if (callback != null) { if (isAcks) { messageIdFuture.addCallback(new RpcFutureCallback() { @Override public void onFailure(Throwable t) { - callback.onCompletion(null, new ExecutionException(t)); + callback.onCompletion(metadata, new ExecutionException(t)); } @Override public void onSuccess(String result) { - callback.onCompletion(null, null); + callback.onCompletion(metadata, null); } }); } else { - callback.onCompletion(null, null); + callback.onCompletion(metadata, null); } } @@ -296,7 +302,9 @@ public static class Builder { private final Serializer keySerializer; private final Serializer valueSerializer; - private int batchSize; + private long batchSize; + private long lingerMs; + private int bufferMemory; private boolean isAcks; private int maxRequestSize; @@ -314,9 +322,11 @@ private void setDefaults() { this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; + this.lingerMs = PubsubProducerConfig.DEFAULT_LINGER_MS; + this.bufferMemory = PubsubProducerConfig.DEFAULT_BUFFER_MEMORY; } - public Builder batchSize(int val) { + public Builder batchSize(long val) { Preconditions.checkArgument(val > 0); batchSize = val; return this; @@ -333,6 +343,18 @@ public Builder maxRequestSize(int val) { return this; } + public Builder lingerMs(long val) { + Preconditions.checkArgument(val >= 0); + lingerMs = val; + return this; + } + + public Builder bufferMemory(int val) { + Preconditions.checkArgument(val >= 0); + bufferMemory = val; + return this; + } + public PubsubProducer build() { return new PubsubProducer(this); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index cbaed929..505fda20 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -38,7 +38,7 @@ public class PubsubProducerConfig extends AbstractConfig { + "the Serializer interface."; public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; - private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for vlaue that " + private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that " + "implements the Serializer interface."; public static final String BATCH_SIZE_CONFIG = "batch.size"; @@ -54,9 +54,17 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; + public static final String LINGER_MS_CONFIG = "linger.ms"; + private static final String LINGER_MS_DOC = "Referred to as delayThreshold in CPS."; + + public static final String BUFFER_MEMORY_CONFIG = "buffer.memory"; + private static final String BUFFER_MEMORY_DOC = "Referred to as maxOutstandingRequestBytes in CPS."; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final long DEFAULT_LINGER_MS = 0L; public static final boolean DEFAULT_ACKS = true; - public static final int DEFAULT_MAX_REQUEST_SIZE = 1*1024*1024; + public static final int DEFAULT_MAX_REQUEST_SIZE = 1024*1024; + public static final int DEFAULT_BUFFER_MEMORY = 32 * 1024 * 1024; static { CONFIG = @@ -66,9 +74,11 @@ public class PubsubProducerConfig extends AbstractConfig { .define( VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(BUFFER_MEMORY_CONFIG, Type.LONG, DEFAULT_BUFFER_MEMORY, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) + .define(LINGER_MS_CONFIG, Type.LONG, DEFAULT_LINGER_MS, atLeast(0L), Importance.MEDIUM, LINGER_MS_DOC) .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index fc02a198..94fd05bd 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -42,6 +42,7 @@ public ProducerThread(String s, Properties props, String topic, int numMessages) this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) + .lingerMs(Long.parseLong(props.getProperty("linger.ms"))) .build(); this.topic = topic; this.numMessages = numMessages; @@ -55,8 +56,8 @@ public void run() { private void processCommand() { try { - ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command); for (int i = 0; i < numMessages; i++) { + ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command + i); producer.send( msg, new Callback() { @@ -65,6 +66,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { log.error("Exception sending the message: " + exception.getMessage()); } else { log.info("Successfully sent message"); + log.info("HERE'S THE METADATA: " + metadata); } } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java index 94478810..f94b3cc3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java @@ -54,7 +54,7 @@ public void uncaughtException(Thread t, Throwable e) { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("acks", "all") - .put("batch.size", "5") + .put("batch.size", "1000") .put("linger.ms", "1") .build() ); From 5d8c5eb376a638c8c915e0fd7d29a8b8b11e0bda Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 12:40:42 -0700 Subject: [PATCH 122/140] Adjusting settings on mapped/CPSPublisherTask. --- .../com/google/pubsub/clients/mapped/CPSPublisherTask.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index d8fa81b8..13a098fe 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -3,6 +3,7 @@ import com.beust.jcommander.JCommander; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.util.Durations; import com.google.pubsub.clients.common.LoadTestRunner; import com.google.pubsub.clients.common.MetricsHandler.MetricName; import com.google.pubsub.clients.common.Task; @@ -36,6 +37,8 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) .batchSize(9500000) + .lingerMs(request.getPublishBatchDuration().getSeconds()) + .bufferMemory((int)Durations.toMillis(request.getPublishBatchDuration())) .isAcks(true) .build(); } @@ -60,7 +63,7 @@ public ListenableFuture doRun() { log.error(exception.getMessage(), exception); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + result.set(RunResult.fromBatchSize(batchSize)); } }); } From 924a0cacf1e1cfadf5b2b5c27777f762045db109 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Tue, 14 Mar 2017 15:45:06 -0700 Subject: [PATCH 123/140] Fixed flush so it properly clears the records of the publishers. --- .../java/com/google/pubsub/clients/producer/PubsubProducer.java | 1 + .../java/com/google/pubsub/clients/tests/ProducerThread.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 8b977abb..f63eeb42 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -251,6 +251,7 @@ public void flush() { log.error("Exception occurred during flush: " + e); } } + publishers.clear(); } /** diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index 94fd05bd..f2d6606a 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -66,7 +66,6 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { log.error("Exception sending the message: " + exception.getMessage()); } else { log.info("Successfully sent message"); - log.info("HERE'S THE METADATA: " + metadata); } } } From 9f4d8df9cd538615bed3ca4715186ad22f2c53ea Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 16 Mar 2017 15:10:27 -0700 Subject: [PATCH 124/140] Modified configs to make load tests equivalent. --- .../pubsub/clients/mapped/CPSPublisherTask.java | 8 ++++---- .../pubsub/clients/producer/PubsubProducer.java | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index 13a098fe..ab31651e 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -36,9 +36,9 @@ private CPSPublisherTask(StartRequest request) { this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), new StringSerializer()) - .batchSize(9500000) - .lingerMs(request.getPublishBatchDuration().getSeconds()) - .bufferMemory((int)Durations.toMillis(request.getPublishBatchDuration())) + .batchSize(9500000L) + .lingerMs(Durations.toMillis(request.getPublishBatchDuration())) + .bufferMemory(1000000000) .isAcks(true) .build(); } @@ -63,7 +63,7 @@ public ListenableFuture doRun() { log.error(exception.getMessage(), exception); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(batchSize)); + result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); } }); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index f63eeb42..100c6c17 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -28,6 +28,7 @@ import com.google.pubsub.common.PubsubChannelUtil; import com.google.pubsub.v1.TopicName; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; @@ -60,7 +61,7 @@ public class PubsubProducer implements Producer { private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 1000; + private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; private final String project; private final Serializer keySerializer; @@ -72,7 +73,7 @@ public class PubsubProducer implements Producer { private final long lingerMs; private final Map publishers; - private boolean closed = false; + private AtomicBoolean closed; private PubsubProducer(Builder builder) { project = builder.project; @@ -84,6 +85,7 @@ private PubsubProducer(Builder builder) { lingerMs = builder.lingerMs; bufferMemory = builder.bufferMemory; publishers = new HashMap<>(); + closed = new AtomicBoolean(false); } public PubsubProducer(Map configs) { @@ -141,6 +143,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); + closed.set(false); log.debug("Producer successfully initialized."); } @@ -157,7 +160,7 @@ public Future send(ProducerRecord record) { */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); - if (closed) { + if (closed.get()) { throw new RuntimeException("Publisher is closed"); } @@ -183,7 +186,6 @@ public Future send(ProducerRecord record, Callback callbac try { Publisher newPub = Publisher.newBuilder(topic) .setBundlingSettings(BundlingSettings.newBuilder() - .setIsEnabled(true) .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) .setDelayThreshold(Duration.millis(lingerMs)) .setRequestByteThreshold(batchSize) @@ -282,6 +284,9 @@ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { throw new IllegalArgumentException("Timeout cannot be negative."); } + if (closed.getAndSet(true)) { + throw new IllegalStateException("Cannot close a producer already closed."); + } for (TopicName topic : publishers.keySet()) { try { publishers.get(topic).shutdown(); @@ -290,7 +295,6 @@ public void close(long timeout, TimeUnit unit) { } } log.debug("Closed producer"); - closed = true; } /** From 864858691f2a909b5f72f4293cc5c38d436ff0a7 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 16 Mar 2017 15:14:22 -0700 Subject: [PATCH 125/140] Fixed closed to use AtomicBoolean. --- .../com/google/pubsub/clients/producer/PubsubProducer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 100c6c17..dd570d5f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -143,8 +143,7 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); publishers = new HashMap<>(); - closed.set(false); - + closed = new AtomicBoolean(false); log.debug("Producer successfully initialized."); } @@ -285,7 +284,7 @@ public void close(long timeout, TimeUnit unit) { throw new IllegalArgumentException("Timeout cannot be negative."); } if (closed.getAndSet(true)) { - throw new IllegalStateException("Cannot close a producer already closed."); + throw new IllegalStateException("Cannot close a producer if already closed."); } for (TopicName topic : publishers.keySet()) { try { From 0d672e2c13c2b441d1d2b16a5e4858ad401a65aa Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 14:40:32 -0700 Subject: [PATCH 126/140] Each PubsubProducer corresponds to a single topic. --- .../clients/producer/PubsubProducer.java | 91 ++++++++++--------- .../producer/PubsubProducerConfig.java | 4 + .../pubsub/clients/tests/ProducerThread.java | 2 +- .../producer/PubsubProducerConfigTest.java | 1 + 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index dd570d5f..6983c346 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -64,6 +64,7 @@ public class PubsubProducer implements Producer { private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; private final String project; + private final String topic; private final Serializer keySerializer; private final Serializer valueSerializer; private final long batchSize; @@ -71,12 +72,13 @@ public class PubsubProducer implements Producer { private final boolean isAcks; private final int maxRequestSize; private final long lingerMs; - private final Map publishers; + private Publisher publisher; private AtomicBoolean closed; private PubsubProducer(Builder builder) { project = builder.project; + topic = builder.topic; batchSize = builder.batchSize; isAcks = builder.isAcks; maxRequestSize = builder.maxRequestSize; @@ -84,7 +86,7 @@ private PubsubProducer(Builder builder) { valueSerializer = builder.valueSerializer; lingerMs = builder.lingerMs; bufferMemory = builder.bufferMemory; - publishers = new HashMap<>(); + publisher = newPublisher(); closed = new AtomicBoolean(false); } @@ -139,14 +141,35 @@ private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer batchSize = configs.getLong(PubsubProducerConfig.BATCH_SIZE_CONFIG); isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); + topic = configs.getString(PubsubProducerConfig.TOPIC_CONFIG); maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); - publishers = new HashMap<>(); + publisher = newPublisher(); closed = new AtomicBoolean(false); log.debug("Producer successfully initialized."); } + private Publisher newPublisher() { + TopicName topicName = TopicName.create(project, topic); + Publisher newPub = null; + try { + newPub = Publisher.newBuilder(topicName) + .setBundlingSettings(BundlingSettings.newBuilder() + .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) + .setDelayThreshold(Duration.millis(lingerMs)) + .setRequestByteThreshold(batchSize) + .build()) + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(bufferMemory) + .build()) + .build(); + } catch (IOException e) { + log.error("Exception occurred: " + e); + } + return newPub; + } + /** * Sends the given record. */ @@ -156,6 +179,7 @@ public Future send(ProducerRecord record) { /** * Sends the given record and invokes the specified callback. + * The given record must have the same topic as the producer. */ public Future send(ProducerRecord record, Callback callback) { log.trace("Received " + record.toString()); @@ -163,52 +187,34 @@ public Future send(ProducerRecord record, Callback callbac throw new RuntimeException("Publisher is closed"); } - TopicName topic = TopicName.create(project, record.topic()); + if (!record.topic().equals(topic)) { + throw new IllegalArgumentException("The record's topic must be the same as the Producer's topic."); + } + Map attributes = new HashMap<>(); byte[] serializedKey = ByteString.EMPTY.toByteArray(); if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic.getTopic(), record.key()); + serializedKey = this.keySerializer.serialize(topic, record.key()); attributes .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); } byte[] valueBytes = ByteString.EMPTY.toByteArray(); if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic.getTopic(), record.value()); + valueBytes = valueSerializer.serialize(topic, record.value()); } checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); - synchronized (publishers) { - if (!publishers.containsKey(topic)) { - try { - Publisher newPub = Publisher.newBuilder(topic) - .setBundlingSettings(BundlingSettings.newBuilder() - .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) - .setDelayThreshold(Duration.millis(lingerMs)) - .setRequestByteThreshold(batchSize) - .build()) - .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(bufferMemory) - .build()) - .build(); - publishers.put(topic, newPub); - } catch (IOException e) { - log.error("Exception occurred: " + e); - } - } - } - PubsubMessage message = PubsubMessage.newBuilder() .setData(ByteString.copyFrom(valueBytes)) .putAllAttributes(attributes) .build(); - - RpcFuture messageIdFuture = publishers.get(topic).publish(message); + RpcFuture messageIdFuture = publisher.publish(message); Future future = SettableFuture.create(); - RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic.getTopic(), 0), + RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic, 0), 0, 0, System.currentTimeMillis(), 0, serializedKey.length, valueBytes.length); if (callback != null) { @@ -245,14 +251,13 @@ private void checkRecordSize(int size) { */ public void flush() { log.debug("Flushing..."); - for (TopicName topic : publishers.keySet()) { - try { - publishers.get(topic).shutdown(); - } catch (Exception e) { - log.error("Exception occurred during flush: " + e); - } + // Shut down publisher to flush the logs, then immediately restart it. + try { + publisher.shutdown(); + publisher = newPublisher(); + } catch (Exception e) { + log.error("Exception occurred during flush: " + e); } - publishers.clear(); } /** @@ -286,12 +291,10 @@ public void close(long timeout, TimeUnit unit) { if (closed.getAndSet(true)) { throw new IllegalStateException("Cannot close a producer if already closed."); } - for (TopicName topic : publishers.keySet()) { - try { - publishers.get(topic).shutdown(); - } catch (Exception e) { - log.error("Exception occurred during close: " + e); - } + try { + publisher.shutdown(); + } catch (Exception e) { + log.error("Exception occurred during close: " + e); } log.debug("Closed producer"); } @@ -305,6 +308,7 @@ public static class Builder { private final String project; private final Serializer keySerializer; private final Serializer valueSerializer; + private final String topic; private long batchSize; private long lingerMs; @@ -312,12 +316,13 @@ public static class Builder { private boolean isAcks; private int maxRequestSize; - public Builder(String project, Serializer keySerializer, Serializer valueSerializer) { + public Builder(String project, String topic, Serializer keySerializer, Serializer valueSerializer) { Preconditions .checkArgument(project != null && keySerializer != null && valueSerializer != null); this.project = project; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; + this.topic = topic; setDefaults(); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java index 505fda20..772ca0fb 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java @@ -51,6 +51,9 @@ public class PubsubProducerConfig extends AbstractConfig { public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project to use with the publisher."; + public static final String TOPIC_CONFIG = "topic"; + private static final String TOPIC_DOC = "Topic to which messages are published."; + public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; @@ -76,6 +79,7 @@ public class PubsubProducerConfig extends AbstractConfig { VALUE_SERIALIZER_CLASS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, DEFAULT_BUFFER_MEMORY, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(TOPIC_CONFIG, Type.STRING, Importance.HIGH, TOPIC_DOC) .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, BATCH_SIZE_DOC) .define(LINGER_MS_CONFIG, Type.LONG, DEFAULT_LINGER_MS, atLeast(0L), Importance.MEDIUM, LINGER_MS_DOC) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java index f2d6606a..95d83999 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java @@ -39,7 +39,7 @@ public class ProducerThread implements Runnable { public ProducerThread(String s, Properties props, String topic, int numMessages) { this.command = s; - this.producer = new Builder<>(props.getProperty("project"), new StringSerializer(), new StringSerializer()) + this.producer = new Builder<>(props.getProperty("project"), topic, new StringSerializer(), new StringSerializer()) .batchSize(Integer.parseInt(props.getProperty("batch.size"))) .isAcks(props.getProperty("acks").matches("1|all")) .lingerMs(Long.parseLong(props.getProperty("linger.ms"))) diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java index 000536cd..4baefb35 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java @@ -24,6 +24,7 @@ public void testSuccessAllConfigsProvided() { .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("project", "unit-test-project") + .put("topic", "unit-test-topic") .build() ); From f94631ea0e2460324bb3c60d8b695d4771f04911 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 16:37:00 -0700 Subject: [PATCH 127/140] Modified mapped/CPSPublisherTask to reflect updates in PubsubProducer. --- .../clients/mapped/CPSPublisherTask.java | 2 +- .../clients/producer/PubsubProducerTest.java | 116 ++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index ab31651e..290d0c48 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -34,7 +34,7 @@ private CPSPublisherTask(StartRequest request) { this.payload = LoadTestRunner.createMessage(request.getMessageSize()); this.batchSize = request.getPublishBatchSize(); - this.publisher = new PubsubProducer.Builder<>(request.getProject(), new StringSerializer(), + this.publisher = new PubsubProducer.Builder<>(request.getProject(), topic, new StringSerializer(), new StringSerializer()) .batchSize(9500000L) .lingerMs(Durations.toMillis(request.getPublishBatchDuration())) diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java new file mode 100644 index 00000000..a56e9d41 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.pubsub.clients.producer; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.powermock.api.easymock.PowerMock.createMock; +import com.google.pubsub.clients.producer.PubsubProducer.Builder; +import com.google.pubsub.common.PubsubChannelUtil; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import java.io.IOException; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +//@RunWith(PowerMockRunner.class) +//@PrepareForTest(PublisherFutureStub.class) +public class PubsubProducerTest { + + private static final String TOPIC = "testTopic"; + private static final String MESSAGE = "testMessage"; + private static final String PROJECT = "unit-test-proj"; + private static final StringSerializer testSerializer = new StringSerializer(); + private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); + + private static PublisherFutureStub mockStub; + @Mock private static PubsubChannelUtil mockChannelUtil; + + private boolean channelIsClosed; + + + @Before + public void setUp() { + // mockStub = createMock(PublisherFutureStub.class); + MockitoAnnotations.initMocks(this); + channelIsClosed = false; + + Mockito.doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + return channelIsClosed = true; + } + }).when(mockChannelUtil).closeChannel(); + } + + /* send() tests */ + @Test + public void testSendPublisherClosed() throws IOException { + /* PubsubProducer testProducer = getNewProducer(); + testProducer.close(); + + try { + testProducer.send(testRecord); + fail("Should have thrown a runtime exception."); + } catch (RuntimeException expected) { + assertTrue("Channel should be closed.", channelIsClosed); + }*/ + } + + @Test + public void testSendRecordTooLarge() { + + } + + @Test + public void testSendBatchFull() { + + } + + /* This happens when the batch size is > 1 and not enough messages are batched */ + @Test + public void testSendMessageNotSentYet() { + + } + + /* flush() tests */ + @Test + public void testFlushMessagesSent() { + + } + + /* close() tests */ + @Test + public void testCloseTimeoutLessThanZero() { + + } + + @Test + public void testCloseChannelCloseSuccessful() { + + } + +} From 5dbe2789e5264e2a61bcf06ed16d1d49caad2a23 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Fri, 17 Mar 2017 16:40:08 -0700 Subject: [PATCH 128/140] Deleted unit test classes that are empty/no longer in use. --- .../clients/producer/PubsubProducerTest.java | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java deleted file mode 100644 index a56e9d41..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.producer; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.powermock.api.easymock.PowerMock.createMock; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import com.google.pubsub.common.PubsubChannelUtil; -import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; -import java.io.IOException; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -//@RunWith(PowerMockRunner.class) -//@PrepareForTest(PublisherFutureStub.class) -public class PubsubProducerTest { - - private static final String TOPIC = "testTopic"; - private static final String MESSAGE = "testMessage"; - private static final String PROJECT = "unit-test-proj"; - private static final StringSerializer testSerializer = new StringSerializer(); - private static final ProducerRecord testRecord = new ProducerRecord(TOPIC, MESSAGE); - - private static PublisherFutureStub mockStub; - @Mock private static PubsubChannelUtil mockChannelUtil; - - private boolean channelIsClosed; - - - @Before - public void setUp() { - // mockStub = createMock(PublisherFutureStub.class); - MockitoAnnotations.initMocks(this); - channelIsClosed = false; - - Mockito.doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - return channelIsClosed = true; - } - }).when(mockChannelUtil).closeChannel(); - } - - /* send() tests */ - @Test - public void testSendPublisherClosed() throws IOException { - /* PubsubProducer testProducer = getNewProducer(); - testProducer.close(); - - try { - testProducer.send(testRecord); - fail("Should have thrown a runtime exception."); - } catch (RuntimeException expected) { - assertTrue("Channel should be closed.", channelIsClosed); - }*/ - } - - @Test - public void testSendRecordTooLarge() { - - } - - @Test - public void testSendBatchFull() { - - } - - /* This happens when the batch size is > 1 and not enough messages are batched */ - @Test - public void testSendMessageNotSentYet() { - - } - - /* flush() tests */ - @Test - public void testFlushMessagesSent() { - - } - - /* close() tests */ - @Test - public void testCloseTimeoutLessThanZero() { - - } - - @Test - public void testCloseChannelCloseSuccessful() { - - } - -} From 297551abcf8cdbfadf44f7fa6572c3aa0b7bd4a0 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Mar 2017 10:19:55 -0700 Subject: [PATCH 129/140] Added NotImplementedException to method missing it. --- .../java/com/google/pubsub/clients/consumer/PubsubConsumer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java index 5e8fd2d8..93cb45df 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java @@ -126,7 +126,7 @@ public void seek(TopicPartition partition, long offset) { } public void seekToBeginning(Collection partitions) { - + throw new NotImplementedException("Not yet implemented"); } public void seekToEnd(Collection partitions) { From 9ae4e5869b39bffdbbd9c217de2d593d92913cf2 Mon Sep 17 00:00:00 2001 From: Amanda Chalfant Date: Thu, 23 Mar 2017 11:27:59 -0700 Subject: [PATCH 130/140] Small fixes on style and error messages. --- .../src/main/java/com/google/pubsub/flic/Driver.java | 8 ++++---- .../java/com/google/pubsub/flic/output/SheetsService.java | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index e1daa3c0..4ae16560 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -134,7 +134,6 @@ public class Driver { names = {"--cps_mapped_java_publisher_count"}, description = "Number of cps mapped publishers to start." ) - private int cpsMappedJavaPublisherCount = 0; @Parameter( @@ -414,9 +413,10 @@ public void run(BiFunction, Controller> contr for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsVtkJavaPublisherCount + cpsMappedJavaPublisherCount - > 0, - "--cps_gcloud_java_publisher or --cps_gcloud_python_publisher must be > 0."); + cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + + cpsVtkJavaPublisherCount + cpsMappedJavaPublisherCount > 0, + "--cps_gcloud_java_publisher, --cps_gcloud_python_publisher, or" + + "--cps_mapped_java_publisher must be > 0."); clientParamsMap.put( new ClientParams(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER, "gcloud-subscription" + i), cpsGcloudJavaSubscriberCount / cpsSubscriptionFanout); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index a60d527a..d8d7acfb 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -138,6 +138,7 @@ public List>> getValuesList(Map>> getValuesList(Map Date: Fri, 18 Aug 2017 14:27:39 -0700 Subject: [PATCH 131/140] Mapped api consumer (#111) * Basic consumer methods - subscribe, unsubscribe, poll, subscription, close * ChannelUtil handling gRPC global channel * Handling user configuration - separate config directory * Future stubs usage * Logging, errors handling * Unit tests, using GrpcServerRule instead of mocks * Fixing travis .yml to include dependecy on mapped-api jar --- .travis.yml | 5 +- pubsub-mapped-api/pom.xml | 114 ++- .../pubsub/clients/config/ConsumerConfig.java | 142 ++++ .../com/google/pubsub/clients/config/LICENSE | 330 ++++++++ .../com/google/pubsub/clients/config/NOTICE | 8 + .../clients/consumer/KafkaConsumer.java | 702 ++++++++++++++++++ .../clients/consumer/PubsubConsumer.java | 191 ----- .../clients/producer/PubsubProducer.java | 20 +- .../com/google/pubsub/common/ChannelUtil.java | 43 ++ .../pubsub/common/PubsubChannelUtil.java | 69 -- .../clients/config/ConsumerConfigTest.java | 42 ++ .../clients/consumer/KafkaConsumerTest.java | 367 +++++++++ 12 files changed, 1692 insertions(+), 341 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java diff --git a/.travis.yml b/.travis.yml index 8c4d9fd0..139eda3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,6 @@ script: - mvn -q -B -f kafka-connector/pom.xml clean verify - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Dmaven.test.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 - - mvn -q -B -f pubsub-mapped-api/pom.xml clean verify - - mvn -q -B -f load-test-framework/pom.xml clean verify + - mvn -q -B -f pubsub-mapped-api/pom.xml clean install verify + - mvn -q -B -f load-test-framework/pom.xml clean install verify + diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index c7144c08..7239c6a4 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -7,6 +7,9 @@ 1.0-SNAPSHOT MappedApi http://maven.apache.org + + 1.3.0 + com.google.cloud @@ -14,54 +17,72 @@ 0.9.4-alpha - junit - junit - 4.12 + com.google.auth + google-auth-library-credentials + 0.7.0 - org.apache.kafka - kafka_2.10 - 0.10.1.1 + io.grpc + grpc-netty + ${grpc.version} - org.powermock - powermock-module-junit4 - 1.6.6 + io.grpc + grpc-auth + ${grpc.version} - org.powermock - powermock-api-mockito - 1.6.6 + io.grpc + grpc-protobuf + ${grpc.version} - org.powermock - powermock-api-easymock - 1.6.6 + io.grpc + grpc-stub + ${grpc.version} + + + io.netty + netty-tcnative-boringssl-static + 1.1.33.Fork26 + + + junit + junit + 4.12 org.apache.commons commons-lang3 3.4 + + com.google.auth + google-auth-library-oauth2-http + RELEASE + org.slf4j slf4j-api - 1.7.21 + RELEASE - org.slf4j - slf4j-log4j12 - 1.7.21 + org.powermock + powermock-module-junit4 + 1.6.5 + test - log4j - log4j - 1.2.17 + io.grpc + grpc-testing + 1.3.0 + test + - org.mockito - mockito-all - 1.9.5 + org.apache.kafka + kafka_2.10 + 0.10.1.1 @@ -74,49 +95,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - - package - - shade - - - false - - - - com.google.pubsub.clients.ClientMain - - - mapped-api - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.5.0 - - com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:0.14.0:exe:${os.detected.classifier} - ${project.basedir}/src/main/proto/ - - - - - compile - compile-custom - - - - org.apache.maven.plugins maven-compiler-plugin diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java new file mode 100644 index 00000000..83cf6cf3 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.pubsub.clients.config; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.serialization.Deserializer; + +//TODO Use builder pattern using Kafka's ConsumerConfig class +/** + * The consumer configuration keys + */ +public class ConsumerConfig extends AbstractConfig { + private static final ConfigDef CONFIG = getInstance(); + + /** + * group.id + */ + public static final String GROUP_ID_CONFIG = "group.id"; + private static final String GROUP_ID_DOC = "A unique string that identifies the consumer group this consumer " + + "belongs to. This property is required if the consumer uses either the group management functionality by using" + + " subscribe(topic) or the Kafka-based offset management strategy."; + + /* + * NOTE: DO NOT CHANGE EITHER CONFIG STRINGS OR THEIR JAVA VARIABLE NAMES AS + * THESE ARE PART OF THE PUBLIC API AND CHANGE WILL BREAK USER CODE. + */ + /** max.poll.records */ + public static final String MAX_POLL_RECORDS_CONFIG = "max.poll.records"; + private static final String MAX_POLL_RECORDS_DOC = + "The maximum number of records returned in a single call to poll() FOR SINGLE TOPIC."; + + /** key.deserializer */ + public static final String KEY_DESERIALIZER_CLASS_CONFIG = "key.deserializer"; + private static final String KEY_DESERIALIZER_CLASS_DOC = + "Deserializer class for key that implements the Deserializer interface."; + + /** value.deserializer */ + public static final String VALUE_DESERIALIZER_CLASS_CONFIG = "value.deserializer"; + private static final String VALUE_DESERIALIZER_CLASS_DOC = + "Deserializer class for value that implements the Deserializer interface."; + + /** subscription.allow.create */ + public static final String SUBSCRIPTION_ALLOW_CREATE_CONFIG = "subscription.allow.create"; + private static final String SUBSCRIPTION_ALLOW_CREATE_DOC = + "Determines if subscriptions for non-existing groups should be created"; + + /** subscription.allow.delete */ + public static final String SUBSCRIPTION_ALLOW_DELETE_CONFIG = "subscription.allow.delete"; + private static final String SUBSCRIPTION_ALLOW_DELETE_DOC = + "Determines if subscriptions for non-existing groups should be created"; + + private static ConfigDef getInstance() { + return new ConfigDef() + .define(GROUP_ID_CONFIG, + Type.STRING, + "", + Importance.HIGH, + GROUP_ID_DOC) + .define(MAX_POLL_RECORDS_CONFIG, + Type.INT, + Importance.MEDIUM, + MAX_POLL_RECORDS_DOC) + .define(SUBSCRIPTION_ALLOW_CREATE_CONFIG, + Type.BOOLEAN, + false, + Importance.MEDIUM, + SUBSCRIPTION_ALLOW_CREATE_DOC) + .define(SUBSCRIPTION_ALLOW_DELETE_CONFIG, + Type.BOOLEAN, + false, + Importance.MEDIUM, + SUBSCRIPTION_ALLOW_DELETE_DOC) + .define(KEY_DESERIALIZER_CLASS_CONFIG, + Type.CLASS, + Importance.HIGH, + KEY_DESERIALIZER_CLASS_DOC) + .define(VALUE_DESERIALIZER_CLASS_CONFIG, + Type.CLASS, + Importance.HIGH, + VALUE_DESERIALIZER_CLASS_DOC); + } + + public static Map addDeserializerToConfig(Map configs, + Deserializer keyDeserializer, Deserializer valueDeserializer) { + Map newConfigs = new HashMap(); + newConfigs.putAll(configs); + if (keyDeserializer != null) + newConfigs.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass()); + if (valueDeserializer != null) + newConfigs.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); + return newConfigs; + } + + public static Properties addDeserializerToConfig(Properties properties, + Deserializer keyDeserializer, Deserializer valueDeserializer) { + Properties newProperties = new Properties(); + newProperties.putAll(properties); + if (keyDeserializer != null) + newProperties.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass().getName()); + if (valueDeserializer != null) + newProperties.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass().getName()); + return newProperties; + } + + public ConsumerConfig(Map props) { + super(CONFIG, props); + } + + ConsumerConfig(Map props, boolean doLog) { + super(CONFIG, props, doLog); + } + + public static Set configNames() { + return CONFIG.names(); + } + + public static void main(String[] args) { + System.out.println(CONFIG.toHtmlTable()); + } + +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE new file mode 100644 index 00000000..354665a7 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE @@ -0,0 +1,330 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +This distribution has a binary dependency on jersey, which is available under the CDDL +License as described below. + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1) +1. Definitions. +1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. “Executable” means the Covered Software in any form other than Source Code. + +1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + +1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. “License” means this document. + +1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. “Modifications” means the Source Code and Executable form of any of the following: + +A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + +B. Any new file that contains any part of the Original Software or previous Modification; or + +C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + +(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + +(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. +3.1. Availability of Source Code. + +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. + +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. + +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. + +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. + +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. + +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. +4.1. New Versions. + +Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. + +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. + +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. + +6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) + +The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE new file mode 100644 index 00000000..5cdd7c60 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE @@ -0,0 +1,8 @@ +Apache Kafka +Copyright 2017 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This distribution has a binary dependency on jersey, which is available under the CDDL +License. The source code of jersey can be found at https://github.com/jersey/jersey/. \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java new file mode 100644 index 00000000..43d29188 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -0,0 +1,702 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + + +package com.google.pubsub.clients.consumer; + +import com.google.api.client.util.Base64; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.Empty; +import com.google.pubsub.clients.config.ConsumerConfig; +import com.google.pubsub.common.ChannelUtil; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.DeleteSubscriptionRequest; +import com.google.pubsub.v1.GetSubscriptionRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ListTopicsResponse; +import com.google.pubsub.v1.PublisherGrpc; +import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SubscriberGrpc; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.Topic; +import io.grpc.CallCredentials; +import io.grpc.Channel; +import io.grpc.Status.Code; +import io.grpc.StatusRuntimeException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KafkaConsumer implements Consumer { + + private static final ConsumerRebalanceListener NO_REBALANCE_LISTENER = null; + private static final Logger log = LoggerFactory.getLogger(KafkaConsumer.class); + private static final String GOOGLE_CLOUD_PROJECT = "projects/" + + System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String SUBSCRIPTION_PREFIX = GOOGLE_CLOUD_PROJECT + "/subscriptions/"; + private static final String TOPIC_PREFIX = GOOGLE_CLOUD_PROJECT + "/topics/"; + + //because of no partition concept, the partition number is constant + private static final int DEFAULT_PARTITION = 0; + + //because checksum is not needed, it is defined as a constant + private static final int DEFAULT_CHECKSUM = 1; + + private static final String KEY_ATTRIBUTE = "key"; + + private final SubscriberFutureStub subscriberFutureStub; + private final PublisherFutureStub publisherFutureStub; + private final Boolean allowSubscriptionCreation; + private final Boolean allowSubscriptionDeletion; + + private final String groupId; + private final int maxPollRecords; + + private ImmutableMap topicNameToSubscription = ImmutableMap.of(); + private ImmutableList topicNames = ImmutableList.of(); + + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + + private int currentPoolIndex; + + public KafkaConsumer(Map configs) { + this(new ConsumerConfig(configs), null, null); + } + + public KafkaConsumer(Map configs, Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig( + ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, valueDeserializer); + } + + public KafkaConsumer(Properties properties) { + this(new ConsumerConfig(properties), null, null); + } + + public KafkaConsumer(Properties properties, Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(new ConsumerConfig( + ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + keyDeserializer, valueDeserializer); + } + + + private KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(configs, + keyDeserializer, + valueDeserializer, + ChannelUtil.getInstance().getChannel(), + ChannelUtil.getInstance().getCallCredentials()); + } + + @SuppressWarnings("unchecked") + KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, + Deserializer valueDeserializer, Channel channel, CallCredentials callCredentials) { + try { + log.debug("Starting PubSub subscriber"); + + Preconditions.checkNotNull(channel); + + SubscriberFutureStub subscriberFutureStub = SubscriberGrpc.newFutureStub(channel); + PublisherFutureStub publisherFutureStub = PublisherGrpc.newFutureStub(channel); + + if(callCredentials != null) { + subscriberFutureStub = subscriberFutureStub.withCallCredentials(callCredentials); + publisherFutureStub = publisherFutureStub.withCallCredentials(callCredentials); + } + + this.subscriberFutureStub = subscriberFutureStub; + this.publisherFutureStub = publisherFutureStub; + + this.keyDeserializer = handleDeserializer(configs, + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer, true); + this.valueDeserializer = handleDeserializer(configs, + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer, false); + this.allowSubscriptionCreation = configs.getBoolean(ConsumerConfig.SUBSCRIPTION_ALLOW_CREATE_CONFIG); + this.allowSubscriptionDeletion = configs.getBoolean(ConsumerConfig.SUBSCRIPTION_ALLOW_DELETE_CONFIG); + this.groupId = configs.getString(ConsumerConfig.GROUP_ID_CONFIG); + //this is a limit on each poll for each topic + this.maxPollRecords = configs.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); + + Preconditions.checkNotNull(this.allowSubscriptionCreation); + Preconditions.checkNotNull(this.allowSubscriptionDeletion); + Preconditions.checkNotNull(this.groupId); + Preconditions.checkArgument(!this.groupId.isEmpty()); + + log.debug("PubSub subscriber created"); + } catch (Throwable t) { + throw new KafkaException("Failed to construct PubSub subscriber", t); + } + } + + private Deserializer handleDeserializer(ConsumerConfig configs, String configString, + Deserializer providedDeserializer, boolean isKey) { + Deserializer deserializer; + if (providedDeserializer == null) { + deserializer = configs.getConfiguredInstance(configString, Deserializer.class); + deserializer.configure(configs.originals(), isKey); + } else { + configs.ignore(configString); + deserializer = providedDeserializer; + } + return deserializer; + } + + @Override + public Set assignment() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Set subscription() { + return topicNameToSubscription.keySet(); + } + + /** + * Subscribe call tries to get existing subscriptions for groupId provided in configuration and topic names. If + * consumer was configured to create subscriptions if they don't exist, on failure with NOT_FOUND status it sends + * "createSubscription" gRPC calls. If it was not configured to create subscriptions, throws KafkaException + * if matching subscriptions do not exist. + */ + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + checkSubscribePreconditions(topics); + + unsubscribe(); + List> futureSubscriptions = deputePubsubSubscribesGet(topics); + Map subscriptionMap = getSubscriptionsFromPubsub(futureSubscriptions); + + topicNameToSubscription = ImmutableMap.copyOf(subscriptionMap); + topicNames = ImmutableList.copyOf(subscriptionMap.keySet()); + currentPoolIndex = 0; + + log.debug("Subscribed to topic(s): {}", Utils.join(topics, ", ")); + } + + private List> deputePubsubSubscribesGet(Collection topics) { + List> responseDatas = new ArrayList<>(); + Set usedNames = new HashSet<>(); + + for (String topic: topics) { + if (!usedNames.contains(topic)) { + String subscriptionString = SUBSCRIPTION_PREFIX + topic + "_" + this.groupId; + ListenableFuture deputedSubscription = + deputeSinglePubsubSubscriptionGet(subscriptionString); + + responseDatas.add(new ResponseData<>(topic, subscriptionString, deputedSubscription)); + usedNames.add(topic); + } + } + return responseDatas; + } + + private ListenableFuture deputeSinglePubsubSubscriptionGet(String subscriptionString) { + return subscriberFutureStub + .getSubscription(GetSubscriptionRequest.newBuilder() + .setSubscription(subscriptionString) + .build()); + } + + private Map getSubscriptionsFromPubsub( + List> responseDatas) { + Map subscriptionMap = new HashMap<>(); + + for (ResponseData responseData: responseDatas) { + boolean success = false; + try { + Subscription s = responseData.getRequestListenableFuture().get(); + subscriptionMap.put(responseData.getTopicName(), s); + success = true; + + } catch (ExecutionException e) { + + if(!shouldTryToCreateSubscription(e)) { + throw new KafkaException(e); + } + //TODO send all needed creation tasks at once rather than block on each of them (future) + Subscription s = tryToCreareSubscription(responseData); + subscriptionMap.put(responseData.getTopicName(), s); + success = true; + + } catch (InterruptedException e) { + throw new InterruptException(e); + } finally { + //if an error is thrown, attempt to delete subscriptions created in this loop + if (!success) + deleteSubscriptionsIfAllowed(subscriptionMap.values()); + } + } + return subscriptionMap; + } + + private Subscription tryToCreareSubscription(ResponseData responseData) { + try { + ListenableFuture subscription = deputeSinglePubsubSubscription( + responseData.getSubscriptionFullName(), responseData.getTopicName()); + return subscription.get(); + } catch (InterruptedException e) { + throw new InterruptException(e); + } catch (ExecutionException e) { + throw new KafkaException(e); + } + } + + private boolean shouldTryToCreateSubscription(ExecutionException e) { + return this.allowSubscriptionCreation && e.getCause() instanceof StatusRuntimeException + && ((StatusRuntimeException)e.getCause()).getStatus().getCode().equals(Code.NOT_FOUND); + } + + private List> deputePubsubSubscribes(Collection topics) { + Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + + List> responseDatas = new ArrayList<>(); + Set usedNames = new HashSet<>(); + + for (String topic: topics) { + if (!usedNames.contains(topic)) { + String subscriptionString = SUBSCRIPTION_PREFIX + topic + "_" + timestamp.getTime() + + "_" + UUID.randomUUID(); + ListenableFuture deputedSubscription = + deputeSinglePubsubSubscription(subscriptionString, topic); + + responseDatas.add(new ResponseData<>(topic, subscriptionString, deputedSubscription)); + usedNames.add(topic); + } + } + return responseDatas; + } + + private ListenableFuture deputeSinglePubsubSubscription(String subscriptionString, + String topicName) { + return subscriberFutureStub + .createSubscription(Subscription.newBuilder() + .setName(subscriptionString) + .setTopic(TOPIC_PREFIX + topicName) + .build()); + } + + private void checkSubscribePreconditions(Collection topics) { + Preconditions.checkArgument(topics != null, + "Topic collection to subscribe to cannot be null"); + + for (String topic: topics) { + Preconditions.checkArgument(topic != null && !topic.trim().isEmpty(), + "Topic collection to subscribe to cannot contain null or empty topic"); + } + } + + @Override + public void subscribe(Collection topics) { + subscribe(topics, NO_REBALANCE_LISTENER); + } + + /** + Subscribe is pulling all topics from PubSub project and subscribing to ones matching provided + pattern. + */ + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + checkPatternSubscribePreconditions(pattern); + + List existingTopics = getPubsubExistingTopics(); + + List matchingTopics = new ArrayList<>(); + + for (Topic topic: existingTopics) { + String topicName = topic.getName().substring(TOPIC_PREFIX.length(), topic.getName().length()); + Matcher m = pattern.matcher(topicName); + if (m.matches()) + matchingTopics.add(topicName); + } + subscribe(matchingTopics); + + log.debug("Subscribed to pattern: {}", pattern); + } + + private List getPubsubExistingTopics() { + ListenableFuture listTopicsResponseListenableFuture = publisherFutureStub + .listTopics(ListTopicsRequest.newBuilder() + .setProject(GOOGLE_CLOUD_PROJECT) + .build()); + + try { + return listTopicsResponseListenableFuture.get().getTopicsList(); + } catch (InterruptedException e) { + throw new InterruptException(e); + } catch (ExecutionException e) { + throw new KafkaException(e); + } + } + + private void checkPatternSubscribePreconditions(Pattern pattern) { + Preconditions.checkArgument(pattern != null, + "Topic pattern to subscribe to cannot be null"); + } + + @Override + public void unsubscribe() { + deleteSubscriptionsIfAllowed(topicNameToSubscription.values()); + topicNameToSubscription = ImmutableMap.of(); + topicNames = ImmutableList.of(); + currentPoolIndex = 0; + } + + private void deleteSubscriptionsIfAllowed(Collection subscriptions) { + if(!allowSubscriptionDeletion) + return; + + List> listenableFutures = new ArrayList<>(); + for (Subscription s: subscriptions) { + ListenableFuture emptyListenableFuture = subscriberFutureStub + .deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setSubscription(s.getName()).build()); + + listenableFutures.add(emptyListenableFuture); + } + + ListenableFuture> listListenableFuture = Futures.allAsList(listenableFutures); + Futures.addCallback(listListenableFuture, new DeleteSubscriptionCallback()); + } + + public ConsumerRecords poll(long timeout) { + checkPollPreconditions(timeout); + + int startedAtIndex = this.currentPoolIndex; + try { + do { + ResponseData pollData = getPullResponseResponseData(timeout); + PullResponse pullResponse = pollData.getRequestListenableFuture().get(); + + List> subscriptionRecords = mapToConsumerRecords(pollData, pullResponse); + + this.currentPoolIndex = (this.currentPoolIndex + 1) % topicNameToSubscription.size(); + + if (!subscriptionRecords.isEmpty()) { + AcknowledgeRequest acknowledgeRequest = getAcknowledgeRequest(pollData.getSubscriptionFullName(), + pullResponse); + //TODO depute acknowledge message with timeout rather than ack immediately + acknowledgeMessage(acknowledgeRequest); + + return getConsumerRecords(pollData, subscriptionRecords); + } + } while (this.currentPoolIndex != startedAtIndex); + + } catch (InterruptedException e) { + throw new InterruptException(e); + } catch (ExecutionException e) { + throw new KafkaException(e); + } + + return new ConsumerRecords<>(new HashMap<>()); + } + + private ConsumerRecords getConsumerRecords(ResponseData pollData, + List> subscriptionRecords) { + Map>> pollRecords = new HashMap<>(); + + TopicPartition topicPartition = new TopicPartition(pollData.getTopicName(), DEFAULT_PARTITION); + pollRecords.put(topicPartition, subscriptionRecords); + + return new ConsumerRecords<>(pollRecords); + } + + private ResponseData getPullResponseResponseData(long timeout) { + String topicName = topicNames.get(this.currentPoolIndex % topicNameToSubscription.size()); + + Subscription subscription = topicNameToSubscription.get(topicName); + ListenableFuture deputedPull = deputeSinglePubsubPull(subscription, timeout); + return new ResponseData<>(topicName, subscription.getName(), deputedPull); + } + + private AcknowledgeRequest getAcknowledgeRequest(String subscription, PullResponse pulled) { + List ackIds = new ArrayList<>(); + for (ReceivedMessage receivedMessage : pulled.getReceivedMessagesList()) { + ackIds.add(receivedMessage.getAckId()); + } + + return AcknowledgeRequest.newBuilder() + .addAllAckIds(ackIds) + .setSubscription(subscription).build(); + } + + private ListenableFuture deputeSinglePubsubPull(Subscription s, long timeout) { + SubscriberFutureStub deadlineFutureStub = + subscriberFutureStub.withDeadlineAfter(timeout, TimeUnit.MILLISECONDS); + + return deadlineFutureStub.pull(PullRequest.newBuilder() + .setSubscription(s.getName()) + .setMaxMessages(this.maxPollRecords) + .setReturnImmediately(true).build()); + } + + private List> mapToConsumerRecords(ResponseData pollData, + PullResponse pulled) { + List> subscriptionRecords = new ArrayList<>(); + + for (ReceivedMessage receivedMessage : pulled.getReceivedMessagesList()) { + ConsumerRecord record = prepareKafkaRecord(receivedMessage, + pollData.getTopicName()); + subscriptionRecords.add(record); + } + + return subscriptionRecords; + } + + private void acknowledgeMessage(AcknowledgeRequest acknowledgeRequest) + throws ExecutionException, InterruptedException { + ListenableFuture acknowledgeFuture = subscriberFutureStub.acknowledge(acknowledgeRequest); + acknowledgeFuture.get(); + } + + private void checkPollPreconditions(long timeout) { + + Preconditions.checkArgument(timeout >= 0, + "Timeout must not be negative"); + + if (topicNameToSubscription.isEmpty()) { + throw new IllegalStateException("Consumer is not subscribed to any topics"); + } + } + + private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, String topic) { + PubsubMessage message = receivedMessage.getMessage(); + + long timestamp = message.getPublishTime().getSeconds(); + TimestampType timestampType = TimestampType.CREATE_TIME; + + //because of no offset concept in PubSub, timestamp is treated as an offset + long offset = timestamp; + + //key of Kafka-style message is stored in PubSub attributes (null possible) + String key = message.getAttributesOrDefault(KEY_ATTRIBUTE, null); + + byte [] deserializedKeyBytes = Base64.decodeBase64(key.getBytes()); + + //lengths of serialized value and serialized key + int serializedValueSize = message.getData().toByteArray().length; + int serializedKeySize = key != null ? deserializedKeyBytes.length : 0; + + V deserializedValue = valueDeserializer.deserialize(topic, message.getData().toByteArray()); + K deserializedKey = key != null ? keyDeserializer.deserialize(topic, deserializedKeyBytes) : null; + + return new ConsumerRecord<>(topic, DEFAULT_PARTITION, offset, timestamp, timestampType, + DEFAULT_CHECKSUM, serializedKeySize, serializedValueSize, deserializedKey, + deserializedValue); + } + + @Override + public void assign(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void commitSync() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void commitSync(final Map offsets) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void commitAsync() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void commitAsync(OffsetCommitCallback callback) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void commitAsync(final Map offsets, + OffsetCommitCallback callback) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void seek(TopicPartition partition, long offset) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void seekToBeginning(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void seekToEnd(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public long position(TopicPartition partition) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Map metrics() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public List partitionsFor(String topic) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Map> listTopics() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void pause(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void resume(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Set paused() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Map offsetsForTimes( + Map timestampsToSearch) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Map beginningOffsets(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Map endOffsets(Collection partitions) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + /** + Temporary subscriptions are being created on subscribe and deleted on replacing subscriptions or + on close. + Closing is necessary to delete any remaining temporary subscriptions from Google Cloud Project. + */ + @Override + public void close() { + log.debug("Closing PubSub subscriber"); + unsubscribe(); + keyDeserializer.close(); + valueDeserializer.close(); + log.debug("PubSub subscriber has been closed"); + } + + @Override + public void wakeup() { + throw new UnsupportedOperationException("Not yet implemented"); + } + + class ResponseData { + private String topicName; + private String subscriptionFullName; + private ListenableFuture requestListenableFuture; + + ResponseData(String topicName, String subscriptionFullName, + ListenableFuture requestListenableFuture) { + this.topicName = topicName; + this.subscriptionFullName = subscriptionFullName; + this.requestListenableFuture = requestListenableFuture; + } + + String getTopicName() { + return topicName; + } + + String getSubscriptionFullName() { + return subscriptionFullName; + } + + ListenableFuture getRequestListenableFuture() { + return requestListenableFuture; + } + } + + class DeleteSubscriptionCallback implements FutureCallback> { + @Override + public void onSuccess(@Nullable List empties) { + + } + + @Override + public void onFailure(Throwable throwable) { + //TODO retry unsubscribe? + log.warn("Failed to unsubscribe to topic", throwable); + } + } + +} + + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java deleted file mode 100644 index 93cb45df..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubsubConsumer.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.consumer; - -import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.OffsetAndTimestamp; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.Deserializer; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -public class PubsubConsumer implements Consumer { - - public PubsubConsumer(Map configs) { - - } - - public PubsubConsumer(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - public PubsubConsumer(Properties properties) { - - } - - public PubsubConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - - } - - public Set assignment() { - throw new NotImplementedException("Not yet implemented"); - } - - public Set subscription() { - throw new NotImplementedException("Not yet implemented"); - } - - public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void subscribe(Collection topics) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { - throw new NotImplementedException("Not yet implemented"); - } - - public void unsubscribe() { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void assign(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public ConsumerRecords poll(long timeout) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void commitSync() { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void commitSync(final Map offsets) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void commitAsync() { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void commitAsync(OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void commitAsync(final Map offsets, - OffsetCommitCallback callback) { - throw new NotImplementedException("Not yet implemented"); - } - - @Override - public void seek(TopicPartition partition, long offset) { - throw new NotImplementedException("Not yet implemented"); - } - - public void seekToBeginning(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public void seekToEnd(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public long position(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - public OffsetAndMetadata committed(TopicPartition partition) { - throw new NotImplementedException("Not yet implemented"); - } - - public Map metrics() { - throw new NotImplementedException("Not yet implemented"); - } - - public List partitionsFor(String topic) { - throw new NotImplementedException("Not yet implemented"); - } - - public Map> listTopics() { - throw new NotImplementedException("Not yet implemented"); - } - - public void pause(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public void resume(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public Set paused() { - throw new NotImplementedException("Not yet implemented"); - } - - public Map offsetsForTimes(Map timestampsToSearch) { - throw new NotImplementedException("Not yet implemented"); - } - - public Map beginningOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public Map endOffsets(Collection partitions) { - throw new NotImplementedException("Not yet implemented"); - } - - public void close() { - - } - - public void close(long timeout, TimeUnit timeUnit) { - throw new NotImplementedException("Not yet implemented"); - } - - public void wakeup() { - throw new NotImplementedException("Not yet implemented"); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java index 6983c346..a00a139b 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java @@ -25,9 +25,16 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.common.PubsubChannelUtil; import com.google.pubsub.v1.TopicName; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.clients.producer.Callback; @@ -46,15 +53,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - /** * A Kafka client that publishes records to Google Cloud Pub/Sub. */ @@ -197,7 +195,7 @@ public Future send(ProducerRecord record, Callback callbac if (record.key() != null) { serializedKey = this.keySerializer.serialize(topic, record.key()); attributes - .put(PubsubChannelUtil.KEY_ATTRIBUTE, new String(serializedKey, StandardCharsets.ISO_8859_1)); + .put("key", new String(serializedKey, StandardCharsets.ISO_8859_1)); } byte[] valueBytes = ByteString.EMPTY.toByteArray(); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java new file mode 100644 index 00000000..a7819428 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java @@ -0,0 +1,43 @@ +package com.google.pubsub.common; + +import com.google.auth.oauth2.GoogleCredentials; +import io.grpc.CallCredentials; +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.auth.MoreCallCredentials; +import java.io.IOException; + +public final class ChannelUtil { + private static final String ENDPOINT = "pubsub.googleapis.com"; + + private static final ChannelUtil INSTANCE = buildInstance(); + + private final ManagedChannel channel; + private final CallCredentials callCredentials; + + private ChannelUtil() throws IOException { + this.callCredentials = MoreCallCredentials.from(GoogleCredentials.getApplicationDefault()); + this.channel = ManagedChannelBuilder.forTarget(ENDPOINT).build(); + } + + private static synchronized ChannelUtil buildInstance() { + try { + return new ChannelUtil(); + } catch (IOException e) { + throw new RuntimeException("Exception while authorising with cloud", e); + } + } + + public static synchronized ChannelUtil getInstance() { + return INSTANCE; + } + + public synchronized Channel getChannel() { + return channel; + } + + public synchronized CallCredentials getCallCredentials() { + return callCredentials; + } +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java deleted file mode 100644 index 062f4d57..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/PubsubChannelUtil.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.common; - -import com.google.auth.oauth2.GoogleCredentials; -import io.grpc.auth.MoreCallCredentials; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.netty.NegotiationType; -import io.grpc.netty.NettyChannelBuilder; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Sets up the pub/sub grpc functionality */ -public class PubsubChannelUtil { - - private static final Logger log = LoggerFactory.getLogger(PubsubChannelUtil.class); - private static final String ENDPOINT = "pubsub.googleapis.com"; - private static final List CPS_SCOPE = Arrays.asList("https://www.googleapis.com/auth/pubsub"); - - public static final String CPS_TOPIC_FORMAT = "projects/%s/topics/%s"; - public static final String KEY_ATTRIBUTE = "key"; - - private ManagedChannel channel; - private CallCredentials callCredentials; - - /* Constructs instance with populated credentials and channel */ - public PubsubChannelUtil() { - GoogleCredentials credentials; - try { - credentials = GoogleCredentials.getApplicationDefault().createScoped(CPS_SCOPE); - } catch (IOException exception) { - log.error("Exception occurred: " + exception.getMessage()); - return; - } - callCredentials = MoreCallCredentials.from(credentials); - channel = NettyChannelBuilder.forAddress(ENDPOINT, 443).negotiationType(NegotiationType.TLS).build(); - } - - public CallCredentials callCredentials() { - return callCredentials; - } - - public Channel channel() { - return channel; - } - - public void closeChannel() { - channel.shutdown(); - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java new file mode 100644 index 00000000..0192f8e0 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java @@ -0,0 +1,42 @@ +package com.google.pubsub.clients.config; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Properties; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.Test; + +public class ConsumerConfigTest { + + @Test + public void checkDeserializersConfiguration() { + Properties properties = new Properties(); + properties.putAll(new ImmutableMap.Builder<>() + .put("key.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer") + .put("value.deserializer", + "org.apache.kafka.common.serialization.IntegerDeserializer") + .put("max.poll.records", 500) + .build() + ); + + ConsumerConfig consumerConfig = new ConsumerConfig(properties); + + Deserializer keyDeserializer = consumerConfig + .getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Deserializer.class); + + Deserializer valueDeserializer = consumerConfig + .getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class); + + assertNotNull(keyDeserializer); + assertEquals(StringDeserializer.class, keyDeserializer.getClass()); + + assertNotNull(valueDeserializer); + assertEquals(IntegerDeserializer.class, valueDeserializer.getClass()); + } + +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java new file mode 100644 index 00000000..2ac8e7a2 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -0,0 +1,367 @@ +package com.google.pubsub.clients.consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.api.client.util.Base64; +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.ByteString; +import com.google.protobuf.Empty; +import com.google.protobuf.Timestamp; +import com.google.pubsub.clients.config.ConsumerConfig; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.DeleteSubscriptionRequest; +import com.google.pubsub.v1.GetSubscriptionRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ListTopicsResponse; +import com.google.pubsub.v1.ListTopicsResponse.Builder; +import com.google.pubsub.v1.PublisherGrpc.PublisherImplBase; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberImplBase; +import com.google.pubsub.v1.Subscription; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcServerRule; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; + + +@RunWith(JUnit4.class) +@SuppressStaticInitializationFor("com.google.pubsub.common.ChannelUtil") +public class KafkaConsumerTest { + + @Rule + public final GrpcServerRule grpcServerRule = new GrpcServerRule().directExecutor(); + + @Test + public void subscribeOnlyGet() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + Set topics = new HashSet<>(Arrays.asList("topic1", "topic2")); + consumer.subscribe(Arrays.asList("topic2", "topic1")); + + Set subscribed = consumer.subscription(); + assertEquals(topics, subscribed); + } + + @Test + public void subscribeNoExistingSubscriptionsAllowCreation() { + KafkaConsumer consumer = getConsumer(true); + grpcServerRule.getServiceRegistry().addService(new SubscriberNoExistingSubscriptionsImpl()); + + Set topics = new HashSet<>(Arrays.asList("topic1", "topic2")); + consumer.subscribe(Arrays.asList("topic2", "topic1")); + + Set subscribed = consumer.subscription(); + assertEquals(topics, subscribed); + } + + @Test + public void subscribeNoExistingSubscriptionsDontAllowCreation() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberNoExistingSubscriptionsImpl()); + + try { + consumer.subscribe(Arrays.asList("topic2", "topic1")); + fail(); + } catch (KafkaException e) { + //expected + } + } + + + @Test + public void unsubscribe() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + consumer.subscribe(Arrays.asList("topic2", "topic1")); + consumer.unsubscribe(); + + assertTrue(consumer.subscription().isEmpty()); + } + + @Test + public void resubscribe() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + Set topics = new HashSet<>(Arrays.asList("topic4", "topic3")); + + consumer.subscribe(Arrays.asList("topic2", "topic1")); + consumer.subscribe(Arrays.asList("topic3", "topic4")); + + Set subscribed = consumer.subscription(); + assertEquals(topics, subscribed); + } + + @Test + public void subscribeWithRecurringTopics() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + Set topics = new HashSet<>(Arrays.asList("topic", "topic1")); + + consumer.subscribe(Arrays.asList("topic", "topic1", "topic")); + + Set subscribed = consumer.subscription(); + assertEquals(topics, subscribed); + } + @Test + public void patternSubscription() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + grpcServerRule.getServiceRegistry().addService(new PublisherImpl()); + + Pattern pattern = Pattern.compile("[a-z]*\\d{3}cat"); + + consumer.subscribe(pattern, null); + Set subscribed = consumer.subscription(); + Set expectedTopics = new HashSet<>(Arrays.asList("thisis123cat", "funnycats000cat")); + assertEquals(expectedTopics, subscribed); + } + + @Test + public void emptyPatternFails() { + KafkaConsumer consumer = getConsumer(false); + try { + Pattern pattern = null; + consumer.subscribe(pattern, null); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Topic pattern to subscribe to cannot be null", e.getMessage()); + } + } + + @Test + public void nullTopicListFails() { + KafkaConsumer consumer = getConsumer(false); + try { + List topics = null; + consumer.subscribe(topics); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Topic collection to subscribe to cannot be null", e.getMessage()); + } + } + + @Test + public void emptyTopicFails() { + KafkaConsumer consumer = getConsumer(false); + try { + List topics = new ArrayList<>(Collections.singletonList(" ")); + consumer.subscribe(topics); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Topic collection to subscribe to cannot contain null or empty topic", e.getMessage()); + } + } + + @Test + public void noSubscriptionsPollFails() { + KafkaConsumer consumer = getConsumer(false); + try { + consumer.poll(100); + fail(); + } catch (IllegalStateException e) { + assertEquals("Consumer is not subscribed to any topics", e.getMessage()); + } + } + + @Test + public void negativePollTimeoutFails() { + KafkaConsumer consumer = getConsumer(false); + try { + consumer.poll(-200); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Timeout must not be negative", e.getMessage()); + } + } + + + @Test + public void deserializeProperly() throws ExecutionException, InterruptedException { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + String topic = "topic"; + Integer key = 125; + String value = "value"; + consumer.subscribe(Collections.singletonList(topic)); + + ConsumerRecords consumerRecord = consumer.poll(100); + + Iterable> recordsForTopic = + consumerRecord.records("topic"); + + ConsumerRecord next = recordsForTopic.iterator().next(); + + assertEquals(consumerRecord.count(), 1); + assertEquals(value, next.value()); + assertEquals(key, next.key()); + } + + @Test + public void pollExecutionException() throws ExecutionException, InterruptedException { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new ErrorSubscriberImpl()); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + try { + consumer.poll(100); + fail(); + } catch (KafkaException e) { + //expected + } + } + + private KafkaConsumer getConsumer(boolean allowesCreation) { + ConsumerConfig consumerConfig = getConsumerConfig(allowesCreation); + return new KafkaConsumer<>(consumerConfig, null, + null, grpcServerRule.getChannel(), null); + } + + private ConsumerConfig getConsumerConfig(boolean allowesCreation) { + Properties properties = new Properties(); + properties.putAll(new ImmutableMap.Builder<>() + .put("key.deserializer", + "org.apache.kafka.common.serialization.IntegerDeserializer") + .put("value.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer") + .put("max.poll.records", 500) + .put("group.id", "groupId") + .put("subscription.allow.create", allowesCreation) + .put("subscription.allow.delete", false) + .build() + ); + + return new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, null, null)); + } + + static class SubscriberGetImpl extends SubscriberImplBase { + + @Override + public void createSubscription(Subscription request, StreamObserver responseObserver) { + responseObserver.onError(new Throwable("This subscriber does not let creating subscriptions")); + } + + @Override + public void getSubscription(GetSubscriptionRequest request, StreamObserver responseObserver) { + Subscription s = Subscription.newBuilder().setName("name").setTopic("projects/null/topics/topic").build(); + responseObserver.onNext(s); + responseObserver.onCompleted(); + } + + @Override + public void deleteSubscription(DeleteSubscriptionRequest request, StreamObserver responseObserver) { + responseObserver.onError(new Throwable("This subscriber does not let deleting subscriptions")); + } + + @Override + public void pull(PullRequest request, StreamObserver responseObserver) { + String topic = "topic"; + Integer key = 125; + String value = "value"; + byte[] serializedKeyBytes = new IntegerSerializer().serialize(topic, key); + String serializedKey = new String(Base64.encodeBase64(serializedKeyBytes)); + byte[] serializedValueBytes = new StringSerializer().serialize(topic, value); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(1500).build(); + + PubsubMessage pubsubMessage = PubsubMessage.newBuilder() + .setPublishTime(timestamp) + .putAttributes("key", serializedKey) + .setData(ByteString.copyFrom(serializedValueBytes)) + .build(); + + ReceivedMessage receivedMessage = ReceivedMessage.newBuilder() + .setMessage(pubsubMessage) + .build(); + + PullResponse pullResponse = PullResponse.newBuilder() + .addReceivedMessages(receivedMessage) + .build(); + + responseObserver.onNext(pullResponse); + responseObserver.onCompleted(); + } + + @Override + public void acknowledge(AcknowledgeRequest request, StreamObserver responseObserver) { + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + } + + static class SubscriberNoExistingSubscriptionsImpl extends SubscriberImplBase { + @Override + public void createSubscription(Subscription request, StreamObserver responseObserver) { + Subscription s = Subscription.newBuilder().setName("name").setTopic("projects/null/topics/topic").build(); + responseObserver.onNext(s); + responseObserver.onCompleted(); + } + + @Override + public void getSubscription(GetSubscriptionRequest request, StreamObserver responseObserver) { + ExecutionException executionException = new ExecutionException(new StatusRuntimeException(Status.NOT_FOUND)); + responseObserver.onError(executionException); + } + } + + static class PublisherImpl extends PublisherImplBase { + + @Override + public void listTopics(ListTopicsRequest request, StreamObserver responseObserver) { + String project = "projects/" + System.getenv("GOOGLE_CLOUD_PROJECT") + "/topics/"; + List topicNames = new ArrayList<>(Arrays.asList( + project + "thisis123cat", project + "abc12345bad", project + "noWay1234", project + "funnycats000cat")); + + Builder listTopicsResponseBuilder = ListTopicsResponse.newBuilder(); + for (String topicName: topicNames) { + listTopicsResponseBuilder.addTopicsBuilder().setName(topicName); + } + + responseObserver.onNext(listTopicsResponseBuilder.build()); + responseObserver.onCompleted(); + } + } + + static class ErrorSubscriberImpl extends SubscriberGetImpl { + + @Override + public void pull(PullRequest request, StreamObserver responseObserver) { + ExecutionException executionException = new ExecutionException(new Throwable("message")); + responseObserver.onError(executionException); + } + + @Override + public void acknowledge(AcknowledgeRequest request, StreamObserver responseObserver) { + responseObserver.onError(new Throwable("This test should not have called ack")); + } + } +} \ No newline at end of file From d674029ea3f6a83d688a7e882fa4026195c3c34d Mon Sep 17 00:00:00 2001 From: pietrzykp Date: Wed, 23 Aug 2017 16:45:25 -0400 Subject: [PATCH 132/140] Configuration refactoring (#118) Configuration for Pubsub-specific options. Consumer configuration object. --- .../pubsub/clients/config/ConsumerConfig.java | 142 -------- .../com/google/pubsub/clients/config/LICENSE | 330 ------------------ .../com/google/pubsub/clients/config/NOTICE | 8 - .../pubsub/clients/consumer/Config.java | 133 +++++++ .../clients/consumer/KafkaConsumer.java | 77 +--- .../consumer/PubSubConsumerConfig.java | 62 ++++ .../consumer/ConsumerConfigCreator.java | 41 +++ .../clients/config/ConsumerConfigTest.java | 42 --- .../pubsub/clients/consumer/ConfigTest.java | 101 ++++++ .../clients/consumer/KafkaConsumerTest.java | 29 +- 10 files changed, 375 insertions(+), 590 deletions(-) delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java create mode 100644 pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java deleted file mode 100644 index 83cf6cf3..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/ConsumerConfig.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.pubsub.clients.config; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.common.serialization.Deserializer; - -//TODO Use builder pattern using Kafka's ConsumerConfig class -/** - * The consumer configuration keys - */ -public class ConsumerConfig extends AbstractConfig { - private static final ConfigDef CONFIG = getInstance(); - - /** - * group.id - */ - public static final String GROUP_ID_CONFIG = "group.id"; - private static final String GROUP_ID_DOC = "A unique string that identifies the consumer group this consumer " - + "belongs to. This property is required if the consumer uses either the group management functionality by using" - + " subscribe(topic) or the Kafka-based offset management strategy."; - - /* - * NOTE: DO NOT CHANGE EITHER CONFIG STRINGS OR THEIR JAVA VARIABLE NAMES AS - * THESE ARE PART OF THE PUBLIC API AND CHANGE WILL BREAK USER CODE. - */ - /** max.poll.records */ - public static final String MAX_POLL_RECORDS_CONFIG = "max.poll.records"; - private static final String MAX_POLL_RECORDS_DOC = - "The maximum number of records returned in a single call to poll() FOR SINGLE TOPIC."; - - /** key.deserializer */ - public static final String KEY_DESERIALIZER_CLASS_CONFIG = "key.deserializer"; - private static final String KEY_DESERIALIZER_CLASS_DOC = - "Deserializer class for key that implements the Deserializer interface."; - - /** value.deserializer */ - public static final String VALUE_DESERIALIZER_CLASS_CONFIG = "value.deserializer"; - private static final String VALUE_DESERIALIZER_CLASS_DOC = - "Deserializer class for value that implements the Deserializer interface."; - - /** subscription.allow.create */ - public static final String SUBSCRIPTION_ALLOW_CREATE_CONFIG = "subscription.allow.create"; - private static final String SUBSCRIPTION_ALLOW_CREATE_DOC = - "Determines if subscriptions for non-existing groups should be created"; - - /** subscription.allow.delete */ - public static final String SUBSCRIPTION_ALLOW_DELETE_CONFIG = "subscription.allow.delete"; - private static final String SUBSCRIPTION_ALLOW_DELETE_DOC = - "Determines if subscriptions for non-existing groups should be created"; - - private static ConfigDef getInstance() { - return new ConfigDef() - .define(GROUP_ID_CONFIG, - Type.STRING, - "", - Importance.HIGH, - GROUP_ID_DOC) - .define(MAX_POLL_RECORDS_CONFIG, - Type.INT, - Importance.MEDIUM, - MAX_POLL_RECORDS_DOC) - .define(SUBSCRIPTION_ALLOW_CREATE_CONFIG, - Type.BOOLEAN, - false, - Importance.MEDIUM, - SUBSCRIPTION_ALLOW_CREATE_DOC) - .define(SUBSCRIPTION_ALLOW_DELETE_CONFIG, - Type.BOOLEAN, - false, - Importance.MEDIUM, - SUBSCRIPTION_ALLOW_DELETE_DOC) - .define(KEY_DESERIALIZER_CLASS_CONFIG, - Type.CLASS, - Importance.HIGH, - KEY_DESERIALIZER_CLASS_DOC) - .define(VALUE_DESERIALIZER_CLASS_CONFIG, - Type.CLASS, - Importance.HIGH, - VALUE_DESERIALIZER_CLASS_DOC); - } - - public static Map addDeserializerToConfig(Map configs, - Deserializer keyDeserializer, Deserializer valueDeserializer) { - Map newConfigs = new HashMap(); - newConfigs.putAll(configs); - if (keyDeserializer != null) - newConfigs.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass()); - if (valueDeserializer != null) - newConfigs.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); - return newConfigs; - } - - public static Properties addDeserializerToConfig(Properties properties, - Deserializer keyDeserializer, Deserializer valueDeserializer) { - Properties newProperties = new Properties(); - newProperties.putAll(properties); - if (keyDeserializer != null) - newProperties.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass().getName()); - if (valueDeserializer != null) - newProperties.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass().getName()); - return newProperties; - } - - public ConsumerConfig(Map props) { - super(CONFIG, props); - } - - ConsumerConfig(Map props, boolean doLog) { - super(CONFIG, props, doLog); - } - - public static Set configNames() { - return CONFIG.names(); - } - - public static void main(String[] args) { - System.out.println(CONFIG.toHtmlTable()); - } - -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE deleted file mode 100644 index 354665a7..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -This distribution has a binary dependency on jersey, which is available under the CDDL -License as described below. - -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1) -1. Definitions. -1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. - -1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. - -1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. - -1.4. “Executable” means the Covered Software in any form other than Source Code. - -1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. - -1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. - -1.7. “License” means this document. - -1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. - -1.9. “Modifications” means the Source Code and Executable form of any of the following: - -A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; - -B. Any new file that contains any part of the Original Software or previous Modification; or - -C. Any new file that is contributed or otherwise made available under the terms of this License. - -1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. - -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. - -1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. - -1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants. -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). - -(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. - -(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. - -3. Distribution Obligations. -3.1. Availability of Source Code. - -Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. - -3.2. Modifications. - -The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. - -3.3. Required Notices. - -You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. - -3.4. Application of Additional Terms. - -You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. - -3.5. Distribution of Executable Versions. - -You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. - -3.6. Larger Works. - -You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. - -4. Versions of the License. -4.1. New Versions. - -Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. - -4.2. Effect of New Versions. - -You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. - -4.3. Modified Versions. - -When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. - -5. DISCLAIMER OF WARRANTY. -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. -6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. - -6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. - -6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. - -6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. -The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. - -9. MISCELLANEOUS. -This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. - -NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) - -The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE deleted file mode 100644 index 5cdd7c60..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/config/NOTICE +++ /dev/null @@ -1,8 +0,0 @@ -Apache Kafka -Copyright 2017 The Apache Software Foundation. - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This distribution has a binary dependency on jersey, which is available under the CDDL -License. The source code of jersey can be found at https://github.com/jersey/jersey/. \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java new file mode 100644 index 00000000..ff95fcd4 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java @@ -0,0 +1,133 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +package com.google.pubsub.clients.consumer; + +import com.google.common.base.Preconditions; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerConfigCreator; +import org.apache.kafka.common.serialization.Deserializer; + +class Config { + + private final Boolean allowSubscriptionCreation; + private final Boolean allowSubscriptionDeletion; + + private final String groupId; + private final int maxPollRecords; + + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + + Config(Map configs) { + this(ConsumerConfigCreator.getConsumerConfig(configs), + new PubSubConsumerConfig(configs), + null, + null); + } + + Config(Map configs, Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(ConsumerConfigCreator.getConsumerConfig( + ConsumerConfig.addDeserializerToConfig( + configs, + keyDeserializer, + valueDeserializer)), + new PubSubConsumerConfig(configs), + keyDeserializer, + valueDeserializer); + } + + Config(Properties properties) { + this(ConsumerConfigCreator.getConsumerConfig(properties), + new PubSubConsumerConfig(properties), + null, + null); + } + + Config(Properties properties, Deserializer keyDeserializer, + Deserializer valueDeserializer) { + this(ConsumerConfigCreator.getConsumerConfig( + ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), + new PubSubConsumerConfig(properties), + keyDeserializer, + valueDeserializer); + } + + private Config(ConsumerConfig consumerConfig, + PubSubConsumerConfig pubSubConsumerConfig, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + + //Kafka-specific options + this.keyDeserializer = handleDeserializer(consumerConfig, + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer, true); + this.valueDeserializer = handleDeserializer(consumerConfig, + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer, false); + this.groupId = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG); + //this is a limit on each poll for each topic + this.maxPollRecords = consumerConfig.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); + + //PubSub-specific options + this.allowSubscriptionCreation = + pubSubConsumerConfig.getBoolean(PubSubConsumerConfig.SUBSCRIPTION_ALLOW_CREATE_CONFIG); + this.allowSubscriptionDeletion = + pubSubConsumerConfig.getBoolean(PubSubConsumerConfig.SUBSCRIPTION_ALLOW_DELETE_CONFIG); + + Preconditions.checkNotNull(this.allowSubscriptionCreation); + Preconditions.checkNotNull(this.allowSubscriptionDeletion); + Preconditions.checkNotNull(this.groupId); + Preconditions.checkArgument(!this.groupId.isEmpty()); + } + + Boolean getAllowSubscriptionCreation() { + return allowSubscriptionCreation; + } + + Boolean getAllowSubscriptionDeletion() { + return allowSubscriptionDeletion; + } + + String getGroupId() { + return groupId; + } + + int getMaxPollRecords() { + return maxPollRecords; + } + + Deserializer getKeyDeserializer() { + return keyDeserializer; + } + + Deserializer getValueDeserializer() { + return valueDeserializer; + } + + private Deserializer handleDeserializer(ConsumerConfig configs, String configString, + Deserializer providedDeserializer, boolean isKey) { + Deserializer deserializer; + if (providedDeserializer == null) { + deserializer = configs.getConfiguredInstance(configString, Deserializer.class); + deserializer.configure(configs.originals(), isKey); + } else { + configs.ignore(configString); + deserializer = providedDeserializer; + } + return deserializer; + } + +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index 43d29188..7e3f371a 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.Empty; -import com.google.pubsub.clients.config.ConsumerConfig; import com.google.pubsub.common.ChannelUtil; import com.google.pubsub.v1.AcknowledgeRequest; import com.google.pubsub.v1.DeleteSubscriptionRequest; @@ -95,57 +94,42 @@ public class KafkaConsumer implements Consumer { private static final String KEY_ATTRIBUTE = "key"; + private final Config config; private final SubscriberFutureStub subscriberFutureStub; private final PublisherFutureStub publisherFutureStub; - private final Boolean allowSubscriptionCreation; - private final Boolean allowSubscriptionDeletion; - - private final String groupId; - private final int maxPollRecords; private ImmutableMap topicNameToSubscription = ImmutableMap.of(); private ImmutableList topicNames = ImmutableList.of(); - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private int currentPoolIndex; public KafkaConsumer(Map configs) { - this(new ConsumerConfig(configs), null, null); + this(new Config<>(configs)); } public KafkaConsumer(Map configs, Deserializer keyDeserializer, Deserializer valueDeserializer) { - this(new ConsumerConfig( - ConsumerConfig.addDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, valueDeserializer); + this(new Config<>(configs, keyDeserializer, valueDeserializer)); } public KafkaConsumer(Properties properties) { - this(new ConsumerConfig(properties), null, null); + this(new Config<>(properties)); } public KafkaConsumer(Properties properties, Deserializer keyDeserializer, Deserializer valueDeserializer) { - this(new ConsumerConfig( - ConsumerConfig.addDeserializerToConfig(properties, keyDeserializer, valueDeserializer)), - keyDeserializer, valueDeserializer); + this(new Config<>(properties, keyDeserializer, valueDeserializer)); } - private KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, - Deserializer valueDeserializer) { - this(configs, - keyDeserializer, - valueDeserializer, + private KafkaConsumer(Config configOptions) { + this(configOptions, ChannelUtil.getInstance().getChannel(), ChannelUtil.getInstance().getCallCredentials()); } @SuppressWarnings("unchecked") - KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, - Deserializer valueDeserializer, Channel channel, CallCredentials callCredentials) { + KafkaConsumer(Config config, Channel channel, CallCredentials callCredentials) { try { log.debug("Starting PubSub subscriber"); @@ -162,20 +146,8 @@ private KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, this.subscriberFutureStub = subscriberFutureStub; this.publisherFutureStub = publisherFutureStub; - this.keyDeserializer = handleDeserializer(configs, - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer, true); - this.valueDeserializer = handleDeserializer(configs, - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer, false); - this.allowSubscriptionCreation = configs.getBoolean(ConsumerConfig.SUBSCRIPTION_ALLOW_CREATE_CONFIG); - this.allowSubscriptionDeletion = configs.getBoolean(ConsumerConfig.SUBSCRIPTION_ALLOW_DELETE_CONFIG); - this.groupId = configs.getString(ConsumerConfig.GROUP_ID_CONFIG); - //this is a limit on each poll for each topic - this.maxPollRecords = configs.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); - - Preconditions.checkNotNull(this.allowSubscriptionCreation); - Preconditions.checkNotNull(this.allowSubscriptionDeletion); - Preconditions.checkNotNull(this.groupId); - Preconditions.checkArgument(!this.groupId.isEmpty()); + //Kafka-specific options + this.config = config; log.debug("PubSub subscriber created"); } catch (Throwable t) { @@ -183,19 +155,6 @@ private KafkaConsumer(ConsumerConfig configs, Deserializer keyDeserializer, } } - private Deserializer handleDeserializer(ConsumerConfig configs, String configString, - Deserializer providedDeserializer, boolean isKey) { - Deserializer deserializer; - if (providedDeserializer == null) { - deserializer = configs.getConfiguredInstance(configString, Deserializer.class); - deserializer.configure(configs.originals(), isKey); - } else { - configs.ignore(configString); - deserializer = providedDeserializer; - } - return deserializer; - } - @Override public Set assignment() { throw new UnsupportedOperationException("Not yet implemented"); @@ -233,7 +192,7 @@ private List> deputePubsubSubscribesGet(Collection deputedSubscription = deputeSinglePubsubSubscriptionGet(subscriptionString); @@ -296,7 +255,7 @@ private Subscription tryToCreareSubscription(ResponseData response } private boolean shouldTryToCreateSubscription(ExecutionException e) { - return this.allowSubscriptionCreation && e.getCause() instanceof StatusRuntimeException + return config.getAllowSubscriptionCreation() && e.getCause() instanceof StatusRuntimeException && ((StatusRuntimeException)e.getCause()).getStatus().getCode().equals(Code.NOT_FOUND); } @@ -396,7 +355,7 @@ public void unsubscribe() { } private void deleteSubscriptionsIfAllowed(Collection subscriptions) { - if(!allowSubscriptionDeletion) + if(!config.getAllowSubscriptionDeletion()) return; List> listenableFutures = new ArrayList<>(); @@ -479,7 +438,7 @@ private ListenableFuture deputeSinglePubsubPull(Subscription s, lo return deadlineFutureStub.pull(PullRequest.newBuilder() .setSubscription(s.getName()) - .setMaxMessages(this.maxPollRecords) + .setMaxMessages(config.getMaxPollRecords()) .setReturnImmediately(true).build()); } @@ -530,8 +489,8 @@ private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, int serializedValueSize = message.getData().toByteArray().length; int serializedKeySize = key != null ? deserializedKeyBytes.length : 0; - V deserializedValue = valueDeserializer.deserialize(topic, message.getData().toByteArray()); - K deserializedKey = key != null ? keyDeserializer.deserialize(topic, deserializedKeyBytes) : null; + V deserializedValue = config.getValueDeserializer().deserialize(topic, message.getData().toByteArray()); + K deserializedKey = key != null ? config.getKeyDeserializer().deserialize(topic, deserializedKeyBytes) : null; return new ConsumerRecord<>(topic, DEFAULT_PARTITION, offset, timestamp, timestampType, DEFAULT_CHECKSUM, serializedKeySize, serializedValueSize, deserializedKey, @@ -649,8 +608,8 @@ public Map endOffsets(Collection partition public void close() { log.debug("Closing PubSub subscriber"); unsubscribe(); - keyDeserializer.close(); - valueDeserializer.close(); + config.getKeyDeserializer().close(); + config.getValueDeserializer().close(); log.debug("PubSub subscriber has been closed"); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java new file mode 100644 index 00000000..0d42f236 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java @@ -0,0 +1,62 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +package com.google.pubsub.clients.consumer; + +import java.util.Map; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; + +//TODO Use builder pattern using Kafka's ConsumerConfig class +/** + * The consumer configuration keys + */ +public class PubSubConsumerConfig extends AbstractConfig { + private static final ConfigDef CONFIG = getInstance(); + + /** subscription.allow.create */ + public static final String SUBSCRIPTION_ALLOW_CREATE_CONFIG = "subscription.allow.create"; + private static final String SUBSCRIPTION_ALLOW_CREATE_DOC = + "Determines if subscriptions for non-existing groups should be created"; + + /** subscription.allow.delete */ + public static final String SUBSCRIPTION_ALLOW_DELETE_CONFIG = "subscription.allow.delete"; + private static final String SUBSCRIPTION_ALLOW_DELETE_DOC = + "Determines if subscriptions for non-existing groups should be created"; + + private static synchronized ConfigDef getInstance() { + return new ConfigDef() + .define(SUBSCRIPTION_ALLOW_CREATE_CONFIG, + Type.BOOLEAN, + false, + Importance.MEDIUM, + SUBSCRIPTION_ALLOW_CREATE_DOC) + .define(SUBSCRIPTION_ALLOW_DELETE_CONFIG, + Type.BOOLEAN, + false, + Importance.MEDIUM, + SUBSCRIPTION_ALLOW_DELETE_DOC); + } + + PubSubConsumerConfig(Map props) { + super(CONFIG, props); + } + + PubSubConsumerConfig(Map props, boolean doLog) { + super(CONFIG, props, doLog); + } + +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java new file mode 100644 index 00000000..0a39446a --- /dev/null +++ b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java @@ -0,0 +1,41 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +package org.apache.kafka.clients.consumer; + +import java.util.Map; +import java.util.Properties; + +/** + * This class is the result of Apache Kafka's ConsumerConfig constructors being package-private. It provides + * a way to instantiate ConsumerConfig objects. Also, it propagates the configuration with options required by + * ConsumerConfig class that are irrelevant for PubSub. + */ +public class ConsumerConfigCreator { + + public static ConsumerConfig getConsumerConfig(Properties properties) { + addDefaultKafkaRequiredOptions(properties); + return new ConsumerConfig(properties); + } + + public static ConsumerConfig getConsumerConfig(Map properties) { + addDefaultKafkaRequiredOptions(properties); + return new ConsumerConfig(properties); + } + + private static void addDefaultKafkaRequiredOptions(Map map) { + if(!map.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) + map.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:8000"); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java deleted file mode 100644 index 0192f8e0..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/config/ConsumerConfigTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.google.pubsub.clients.config; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -import com.google.common.collect.ImmutableMap; -import java.util.Properties; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.IntegerDeserializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.junit.Test; - -public class ConsumerConfigTest { - - @Test - public void checkDeserializersConfiguration() { - Properties properties = new Properties(); - properties.putAll(new ImmutableMap.Builder<>() - .put("key.deserializer", - "org.apache.kafka.common.serialization.StringDeserializer") - .put("value.deserializer", - "org.apache.kafka.common.serialization.IntegerDeserializer") - .put("max.poll.records", 500) - .build() - ); - - ConsumerConfig consumerConfig = new ConsumerConfig(properties); - - Deserializer keyDeserializer = consumerConfig - .getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Deserializer.class); - - Deserializer valueDeserializer = consumerConfig - .getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class); - - assertNotNull(keyDeserializer); - assertEquals(StringDeserializer.class, keyDeserializer.getClass()); - - assertNotNull(valueDeserializer); - assertEquals(IntegerDeserializer.class, valueDeserializer.getClass()); - } - -} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java new file mode 100644 index 00000000..d7341235 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java @@ -0,0 +1,101 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +package com.google.pubsub.clients.consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.DoubleDeserializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.Test; + +public class ConfigTest { + + @Test + public void checkPropertiesDeserializersConfiguration() { + Config config = new Config(getProperties()); + + assertNotNull(config.getKeyDeserializer()); + assertEquals(StringDeserializer.class, config.getKeyDeserializer().getClass()); + + assertNotNull(config.getValueDeserializer()); + assertEquals(IntegerDeserializer.class, config.getValueDeserializer().getClass()); + } + + @Test + public void checkPropertiesDeserializersConfigurationDeserializersProvided() { + Config config = new Config<>(getProperties(), new DoubleDeserializer(), new ByteArrayDeserializer()); + + assertNotNull(config.getKeyDeserializer()); + assertEquals(DoubleDeserializer.class, config.getKeyDeserializer().getClass()); + + assertNotNull(config.getValueDeserializer()); + assertEquals(ByteArrayDeserializer.class, config.getValueDeserializer().getClass()); + } + + @Test + public void checkConfigMapDeserializersConfiguration() { + Config config = new Config(getConfigMap()); + + assertNotNull(config.getKeyDeserializer()); + assertEquals(StringDeserializer.class, config.getKeyDeserializer().getClass()); + + assertNotNull(config.getValueDeserializer()); + assertEquals(IntegerDeserializer.class, config.getValueDeserializer().getClass()); + } + + @Test + public void checkConfigMapDeserializersConfigurationDeserializersProvided() { + Config config = new Config<>(getConfigMap(), new LongDeserializer(), new StringDeserializer()); + + assertNotNull(config.getKeyDeserializer()); + assertEquals(LongDeserializer.class, config.getKeyDeserializer().getClass()); + + assertNotNull(config.getValueDeserializer()); + assertEquals(StringDeserializer.class, config.getValueDeserializer().getClass()); + } + + private Map getConfigMap() { + Map properties = new HashMap<>(); + properties.putAll(getTestOptionsMap()); + return properties; + } + + private Properties getProperties() { + Properties properties = new Properties(); + properties.putAll(getTestOptionsMap()); + return properties; + } + + private ImmutableMap getTestOptionsMap() { + return new ImmutableMap.Builder<>() + .put("key.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer") + .put("value.deserializer", + "org.apache.kafka.common.serialization.IntegerDeserializer") + .put("max.poll.records", 500) + .put("group.id", "groupId") + .put("subscription.allow.create", false) + .put("subscription.allow.delete", false) + .build(); + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java index 2ac8e7a2..89cfe6f4 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -1,3 +1,17 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + package com.google.pubsub.clients.consumer; import static org.junit.Assert.assertEquals; @@ -9,7 +23,6 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Empty; import com.google.protobuf.Timestamp; -import com.google.pubsub.clients.config.ConsumerConfig; import com.google.pubsub.v1.AcknowledgeRequest; import com.google.pubsub.v1.DeleteSubscriptionRequest; import com.google.pubsub.v1.GetSubscriptionRequest; @@ -92,7 +105,6 @@ public void subscribeNoExistingSubscriptionsDontAllowCreation() { } } - @Test public void unsubscribe() { KafkaConsumer consumer = getConsumer(false); @@ -130,6 +142,7 @@ public void subscribeWithRecurringTopics() { Set subscribed = consumer.subscription(); assertEquals(topics, subscribed); } + @Test public void patternSubscription() { KafkaConsumer consumer = getConsumer(false); @@ -202,7 +215,6 @@ public void negativePollTimeoutFails() { } } - @Test public void deserializeProperly() throws ExecutionException, InterruptedException { KafkaConsumer consumer = getConsumer(false); @@ -241,12 +253,12 @@ public void pollExecutionException() throws ExecutionException, InterruptedExcep } private KafkaConsumer getConsumer(boolean allowesCreation) { - ConsumerConfig consumerConfig = getConsumerConfig(allowesCreation); - return new KafkaConsumer<>(consumerConfig, null, - null, grpcServerRule.getChannel(), null); + Properties properties = getTestProperties(allowesCreation); + Config configOptions = new Config(properties); + return new KafkaConsumer<>(configOptions, grpcServerRule.getChannel(), null); } - private ConsumerConfig getConsumerConfig(boolean allowesCreation) { + private Properties getTestProperties(boolean allowesCreation) { Properties properties = new Properties(); properties.putAll(new ImmutableMap.Builder<>() .put("key.deserializer", @@ -259,8 +271,7 @@ private ConsumerConfig getConsumerConfig(boolean allowesCreation) { .put("subscription.allow.delete", false) .build() ); - - return new ConsumerConfig(ConsumerConfig.addDeserializerToConfig(properties, null, null)); + return properties; } static class SubscriberGetImpl extends SubscriberImplBase { From 08c7dd53e457bfa957dd593b7de739a053f4495a Mon Sep 17 00:00:00 2001 From: gewillovic Date: Fri, 25 Aug 2017 15:22:59 -0400 Subject: [PATCH 133/140] Mapped API Producer (#114) * Kafka's Producer implementation based on PubSub's client library, mapping most of the exceptions, but missing some logging stuff. * Deleted some dependencies and added others. * Fixed style, added a missing license, removed unnecessary initializations, fixed key encoding, edited send() return value, removed the driver class. * Removed "test" prefix from all unit tests, supporting request timeout config now. * Supported client.id/buffer.memory configs. * Supported interceptors. * Fixed flic's mapped task, altered travis script to install before verify. * Added a simple inner class implementation for an interceptor, implemented some tests - sufficient for now. * Refactored producer configs, isolating Kafka's from CPS's. * Fixed style, supported offsets, added version and offset to attributes, added safety checks for attributes. * Fixed conflicts after merge. * Changed namings for configs classes and an attribute. * Some changes to driver/mapped task for flic. * The producer is thread safe and has minimal blocking. --- .../clients/mapped/CPSPublisherTask.java | 45 +- .../java/com/google/pubsub/flic/Driver.java | 10 +- pubsub-mapped-api/pom.xml | 143 +++--- .../pubsub/clients/producer/Config.java | 78 ++++ .../clients/producer/KafkaProducer.java | 424 ++++++++++++++++++ .../clients/producer/PubsubProducer.java | 370 --------------- .../producer/PubsubProducerConfig.java | 120 ----- .../pubsub/clients/tests/ProducerThread.java | 84 ---- .../clients/tests/ProducerThreadPool.java | 89 ---- .../producer/ProducerConfigAdapter.java | 45 ++ .../pubsub/clients/producer/ConfigTest.java | 97 ++++ .../clients/producer/KafkaProducerTest.java | 317 +++++++++++++ .../producer/PubsubProducerConfigTest.java | 60 --- 13 files changed, 1082 insertions(+), 800 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java delete mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java create mode 100644 pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java delete mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index 290d0c48..2890aae0 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -1,17 +1,21 @@ package com.google.pubsub.clients.mapped; import com.beust.jcommander.JCommander; -import com.google.common.util.concurrent.ListenableFuture; + import com.google.common.util.concurrent.SettableFuture; -import com.google.protobuf.util.Durations; -import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler.MetricName; +import com.google.common.util.concurrent.ListenableFuture; + import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.producer.PubsubProducer; +import com.google.pubsub.clients.common.LoadTestRunner; +import com.google.pubsub.clients.producer.KafkaProducer; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; +import com.google.pubsub.clients.common.MetricsHandler.MetricName; + +import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,41 +26,46 @@ public class CPSPublisherTask extends Task { private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); + private final String topic; private final String payload; private final int batchSize; - private final PubsubProducer publisher; + private final KafkaProducer publisher; @SuppressWarnings("unchecked") private CPSPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); + this.topic = request.getTopic(); - this.payload = LoadTestRunner.createMessage(request.getMessageSize()); this.batchSize = request.getPublishBatchSize(); + this.payload = LoadTestRunner.createMessage(request.getMessageSize()); + + Properties props = new Properties(); + props.put("project", "dataproc-kafka-test"); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); - this.publisher = new PubsubProducer.Builder<>(request.getProject(), topic, new StringSerializer(), - new StringSerializer()) - .batchSize(9500000L) - .lingerMs(Durations.toMillis(request.getPublishBatchDuration())) - .bufferMemory(1000000000) - .isAcks(true) - .build(); + this.publisher = new KafkaProducer<>(props); } public static void main(String[] args) throws Exception { LoadTestRunner.Options options = new LoadTestRunner.Options(); + new JCommander(options, args); + LoadTestRunner.run(options, CPSPublisherTask::new); } @Override public ListenableFuture doRun() { SettableFuture result = SettableFuture.create(); + AtomicInteger messagesToSend = new AtomicInteger(batchSize); AtomicInteger messagesSentSuccess = new AtomicInteger(batchSize); + for (int i = 0; i < batchSize; i++) { publisher.send( - new ProducerRecord<>(topic, null, System.currentTimeMillis(), null, payload), + new ProducerRecord<>(topic, 0, "", payload), (metadata, exception) -> { if (exception != null) { messagesSentSuccess.decrementAndGet(); @@ -71,5 +80,7 @@ public ListenableFuture doRun() { } @Override - public void shutdown() { publisher.close(); } + public void shutdown() { + publisher.close(); + } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 2e94b772..fc2aa02d 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -137,8 +137,8 @@ public class Driver { private int kafkaSubscriberCount = 0; @Parameter( - names = {"--cps_mapped_java_publisher_count"}, - description = "Number of cps mapped publishers to start." + names = {"--cps_mapped_java_publisher_count"}, + description = "Number of cps mapped publishers to start." ) private int cpsMappedJavaPublisherCount = 0; @@ -171,9 +171,9 @@ public class Driver { private int publishBatchSize = 10; @Parameter( - names = {"--publish_batch_duration"}, - description = "The maximum duration to wait for more messages to batch together.", - converter = DurationConverter.class + names = {"--publish_batch_duration"}, + description = "The maximum duration to wait for more messages to batch together.", + converter = DurationConverter.class ) private Duration publishBatchDuration = Durations.fromMillis(0); diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 7239c6a4..faa6ba4a 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -5,105 +5,138 @@ pubsub-mapped-api jar 1.0-SNAPSHOT + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + MappedApi http://maven.apache.org - - 1.3.0 - - - com.google.cloud - google-cloud-pubsub - 0.9.4-alpha - - - com.google.auth - google-auth-library-credentials - 0.7.0 - + + + Consumer's + + io.grpc grpc-netty - ${grpc.version} + 1.3.0 io.grpc grpc-auth - ${grpc.version} + 1.3.0 io.grpc grpc-protobuf - ${grpc.version} + 1.3.0 io.grpc grpc-stub - ${grpc.version} + 1.3.0 + + + io.grpc + grpc-testing + 1.3.0 + test io.netty netty-tcnative-boringssl-static 1.1.33.Fork26 + + + Producer's + + - junit - junit - 4.12 + com.google.cloud + google-cloud-pubsub + 0.20.3-beta + + + com.google.auth + google-auth-library-oauth2-http + + + com.google.auth + google-auth-library-credentials + + - org.apache.commons - commons-lang3 - 3.4 + org.apache.kafka + kafka-clients + 0.10.1.1 com.google.auth google-auth-library-oauth2-http - RELEASE + 0.7.1 - org.slf4j - slf4j-api - RELEASE + com.google.auth + google-auth-library-credentials + 0.7.1 org.powermock powermock-module-junit4 - 1.6.5 + 1.7.0 test - io.grpc - grpc-testing - 1.3.0 + org.powermock + powermock-api-mockito + 1.7.0 + + + org.mockito + mockito-core + 1.10.19 test + + Shared + + - org.apache.kafka - kafka_2.10 - 0.10.1.1 + junit + junit + 4.12 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.slf4j + slf4j-api + 1.7.21 + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + log4j + log4j + 1.2.17 - - - - - kr.motd.maven - os-maven-plugin - 1.5.0.Final - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.6.1 - - 1.8 - 1.8 - - - - diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java new file mode 100644 index 00000000..1f289a40 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java @@ -0,0 +1,78 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import java.util.Map; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerConfigAdapter; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigDef.Range; +import org.apache.kafka.common.config.ConfigDef.Importance; + +/** + * Provides the configurations for a Kafka Producer instance. + */ +public class Config { + + private ProducerConfig kafkaConfigs; + private PubSubProducerConfig additionalConfigs; + + Config(Map configs) { + additionalConfigs = new PubSubProducerConfig(configs); + kafkaConfigs = ProducerConfigAdapter.getConsumerConfig(configs); + } + + ProducerConfig getKafkaConfigs() { + return kafkaConfigs; + } + + PubSubProducerConfig getAdditionalConfigs() { + return additionalConfigs; + } + + public static class PubSubProducerConfig extends AbstractConfig { + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project that we will connect to."; + + public static final String ELEMENTS_COUNT_CONFIG = "element.count"; + private static final String ELEMENTS_COUNT_DOC = "This configuration controls the default count of" + + " elements in a batch."; + + public static final String AUTO_CREATE_TOPICS_CONFIG = "auto.create.topics.enable"; + private static final String AUTO_CREATE_TOPICS_DOC = "When true topics are automatically created" + + " if they don't exist."; + + private static final ConfigDef CONFIG = new ConfigDef() + .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) + .define(AUTO_CREATE_TOPICS_CONFIG, Type.BOOLEAN, true, Importance.MEDIUM, AUTO_CREATE_TOPICS_DOC) + .define(ELEMENTS_COUNT_CONFIG, Type.LONG, 1000L, Range.atLeast(1L), Importance.MEDIUM, ELEMENTS_COUNT_DOC); + + PubSubProducerConfig(Map originals, boolean doLog) { + super(CONFIG, originals, doLog); + } + + PubSubProducerConfig(Map originals) { + super(CONFIG, originals); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java new file mode 100644 index 00000000..0e0e9ffc --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java @@ -0,0 +1,424 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.client.util.Base64; +import com.google.api.core.ApiFutureCallback; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; + +import com.google.cloud.pubsub.v1.Publisher; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.SettableFuture; + +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.TopicName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.clients.producer.Config.PubSubProducerConfig; + +import java.io.IOException; + +import org.joda.time.DateTime; +import org.threeten.bp.Duration; + +import java.util.Map; +import java.util.List; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.SerializationException; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerInterceptor; +import org.apache.kafka.clients.producer.ProducerConfigAdapter; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; + +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A Kafka client that publishes records to Google Cloud Pub/Sub. + */ +public class KafkaProducer implements Producer { + + private static final long VERSION = 1L; + private static final double MULTIPLIER = 1.0; + private static final AtomicInteger CLIENT_ID = new AtomicInteger(1); + + private Config producerConfig; + private Map publishers; + + private final String project; + private final Serializer keySerializer; + private final Serializer valueSerializer; + + private final int retries; + private final int timeout; + private final int batchSize; + private final long lingerMs; + private final long retriesMs; + private final long elementCount; + private final long bufferMemorySize; + private final boolean isAcks; + private final boolean autoCreate; + private final String clientId; + private final ProducerInterceptors interceptors; + + private final AtomicBoolean closed; + + public KafkaProducer(Map configs) { + this(new Config(configs), null, null); + } + + public KafkaProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { + this(new Config(ProducerConfig.addSerializerToConfig( + configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); + } + + public KafkaProducer(Properties properties) { + this(new Config(properties), null, null); + } + + public KafkaProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { + this(new Config(ProducerConfig.addSerializerToConfig( + properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); + } + + @VisibleForTesting + @SuppressWarnings("unchecked") + public KafkaProducer(Config configs, Serializer keySerializer, Serializer valueSerializer) { + + producerConfig = configs; + + // CPS's configs + project = configs.getAdditionalConfigs().getString(PubSubProducerConfig.PROJECT_CONFIG); + autoCreate = configs.getAdditionalConfigs().getBoolean(PubSubProducerConfig.AUTO_CREATE_TOPICS_CONFIG); + elementCount = configs.getAdditionalConfigs().getLong(PubSubProducerConfig.ELEMENTS_COUNT_CONFIG); + + // Kafka's configs + if (keySerializer == null) { + this.keySerializer = configs.getKafkaConfigs().getConfiguredInstance( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + + this.keySerializer.configure(configs.getKafkaConfigs().originals(), true); + } else { + configs.getKafkaConfigs().ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = configs.getKafkaConfigs().getConfiguredInstance( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + + this.valueSerializer.configure(configs.getKafkaConfigs().originals(), false); + } else { + configs.getKafkaConfigs().ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + closed = new AtomicBoolean(false); + retries = configs.getKafkaConfigs().getInt(ProducerConfig.RETRIES_CONFIG); + timeout = configs.getKafkaConfigs().getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + retriesMs = configs.getKafkaConfigs().getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + lingerMs = configs.getKafkaConfigs().getLong(ProducerConfig.LINGER_MS_CONFIG); + batchSize = configs.getKafkaConfigs().getInt(ProducerConfig.BATCH_SIZE_CONFIG); + isAcks = configs.getKafkaConfigs().getString(ProducerConfig.ACKS_CONFIG).matches("(-)?1|all"); + bufferMemorySize = configs.getKafkaConfigs().getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + + String Id = configs.getKafkaConfigs().getString(ProducerConfig.CLIENT_ID_CONFIG); + if (Id.length() <= 0) { + clientId = "producer-" + CLIENT_ID.getAndIncrement(); + } else { + clientId = Id; + } + + configs.getKafkaConfigs().originals().put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = + (List) (ProducerConfigAdapter.getConsumerConfig(configs.getKafkaConfigs().originals())).getConfiguredInstances( + ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, ProducerInterceptor.class); + this.interceptors = + interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); + + publishers = new ConcurrentHashMap<>(); + } + + private Publisher createPublisher(String topic) { + if (closed.get()) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + Publisher pub = null; + TopicName topicName = null; + TopicAdminClient topicAdmin = null; + + try { + topicName = TopicName.create(project, topic); + topicAdmin = TopicAdminClient.create(); + topicAdmin.getTopic(topicName); + } catch (Exception e) { + if (e.getMessage().contains("NOT_FOUND") && autoCreate) { + topicAdmin.createTopic(topicName); + } else { + throw new KafkaException("Failed to construct kafka producer, Topic not found.", e); + } + } + + try { + pub = Publisher.defaultBuilder(topicName) + + .setBatchingSettings(BatchingSettings.newBuilder() + .setElementCountThreshold(elementCount) + .setRequestByteThreshold((long) batchSize) + .setDelayThreshold(Duration.ofMillis(lingerMs)) + + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(bufferMemorySize) + .build()) + + .build()) + + .setRetrySettings(RetrySettings.newBuilder() + .setMaxAttempts(retries) + .setRetryDelayMultiplier(MULTIPLIER) + .setInitialRetryDelay(Duration.ofMillis(retriesMs)) + .setMaxRetryDelay(Duration.ofMillis((retries + 1) * retriesMs)) + + .setRpcTimeoutMultiplier(MULTIPLIER) + .setInitialRpcTimeout(Duration.ofMillis(timeout)) + .setMaxRpcTimeout(Duration.ofMillis((retries + 1) * timeout)) + + .setTotalTimeout(Duration.ofMillis((retries + 2) * timeout)) + .build()) + + .build(); + } catch (IOException e) { + throw new KafkaException("Failed to construct kafka producer", e); + } + return pub; + } + + /** + * Sends the given record. + */ + public Future send(ProducerRecord originalRecord) { + return send(originalRecord, null); + } + + //TODO: there is an onSendError() in the interceptors, while there are no errors. + + /** + * Sends the given record and invokes the specified callback. + * The given record must have the same topic as the producer. + */ + public Future send(ProducerRecord originalRecord, Callback callback) { + if (closed.get()) { + throw new IllegalStateException("Cannot send after the producer is closed."); + } + + ProducerRecord record = + this.interceptors == null ? originalRecord : this.interceptors.onSend(originalRecord); + + if (record == null) { + throw new NullPointerException(); + } + + DateTime dateTime = new DateTime(); + + byte[] valueBytes = null; + if (record.value() != null) { + try { + valueBytes = valueSerializer.serialize(record.topic(), record.value()); + } catch (ClassCastException e) { + throw new SerializationException("Can't convert value of class " + + record.value().getClass().getName() + " to class " + + producerConfig.getKafkaConfigs().getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + " specified in value.serializer"); + } + } + + if (valueBytes == null || valueBytes.length == 0) { + throw new NullPointerException("Value cannot be null or an empty string."); + } + + byte[] keyBytes = null; + if (record.key() != null) { + try { + keyBytes = keySerializer.serialize(record.topic(), record.key()); + } catch (ClassCastException cce) { + throw new SerializationException("Can't convert key of class " + + record.key().getClass().getName() + " to class " + + producerConfig.getKafkaConfigs().getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + " specified in key.serializer"); + } + } + + PubsubMessage msg = createMessage(record, dateTime.getMillis(), keyBytes, valueBytes); + + ApiFuture messageIdFuture = publishers.computeIfAbsent( + record.topic(), topic -> createPublisher(topic)).publish(msg); + + final Callback cb = callback; + final String topic = record.topic(); + final int keySize = keyBytes == null ? 0 : keyBytes.length, valueSize = valueBytes.length; + + final SettableFuture future = SettableFuture.create(); + + if (callback != null) { + if (isAcks) { + ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { + private RecordMetadata recordMetadata = + getRecordMetadata(topic, dateTime.getMillis(), keySize, valueSize); + + @Override + public void onFailure(Throwable t) { + callbackOnCompletion(cb, recordMetadata, new ExecutionException(t)); + future.setException(t); + } + + @Override + public void onSuccess(String result) { + callbackOnCompletion(cb, recordMetadata, null); + future.set(recordMetadata); + } + }); + } else { + RecordMetadata recordMetadata = + getRecordMetadata(topic, -1L, keySize, valueSize); + callbackOnCompletion(cb, recordMetadata, null); + future.set(recordMetadata); + } + } + + return future; + } + + // Attribute's value's size shouldn't exceed 1024 bytes (256 for key) + private PubsubMessage createMessage(ProducerRecord record, Long dateTime, byte[] key, byte[] value) { + Map attributes = new HashMap<>(); + + attributes.put("id", clientId); + + attributes.put("offset", Long.toString(dateTime)); + + attributes.put("version", Long.toString(VERSION)); + + attributes.put("key", key == null ? "" : new String(Base64.encodeBase64(key))); + + if (attributes.get("key").length() > 1024) { + throw new SerializationException("Key size should be at most 1024 bytes."); + } + + return PubsubMessage.newBuilder().setData(ByteString.copyFrom(value)).putAllAttributes(attributes).build(); + } + + private void callbackOnCompletion(Callback cb, RecordMetadata m, Exception e) { + if (interceptors != null) { + interceptors.onAcknowledgement(m, e); + } + cb.onCompletion(m, e); + } + + private RecordMetadata getRecordMetadata(String topic, long offset, int keySize, int valueSize) { + return new RecordMetadata(new TopicPartition(topic, 0), + offset, 0L, System.currentTimeMillis(), 0L, keySize, valueSize); + } + + /** + * Flushes records that have accumulated. + */ + public void flush() { + Map tempPublishers = publishers; + publishers = new ConcurrentHashMap<>(); + + try { + for (Entry pub : tempPublishers.entrySet()) { + pub.getValue().shutdown(); + } + } catch (Exception e) { + throw new InterruptException("Flush interrupted.", (InterruptedException) e); + } + } + + public List partitionsFor(String topic) { + Node[] dummy = {new Node(0, "", 0)}; + return Arrays.asList(new PartitionInfo(topic, 0, dummy[0], dummy, dummy)); + } + + public Map metrics() { + return new HashMap<>(); + } + + /** + * Closes the producer. + */ + public void close() { + close(0, null); + } + + /** + * Closes the producer with the given timeout. + */ + public void close(long timeout, TimeUnit unit) { + if (timeout < 0) { + throw new IllegalArgumentException("The timeout cannot be negative."); + } + + if (closed.getAndSet(true)) { + throw new IllegalStateException("Cannot close a producer if already closed."); + } + + Map tempPublishers = publishers; + publishers = new ConcurrentHashMap<>(); + + try { + for (Entry pub : tempPublishers.entrySet()) { + pub.getValue().shutdown(); + } + + if (interceptors != null) { + interceptors.close(); + } + + keySerializer.close(); + valueSerializer.close(); + } catch (Exception e) { + throw new KafkaException("Failed to close kafka producer", e); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java deleted file mode 100644 index a00a139b..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducer.java +++ /dev/null @@ -1,370 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.producer; - -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.api.gax.core.RpcFuture; -import com.google.api.gax.core.RpcFutureCallback; -import com.google.api.gax.grpc.BundlingSettings; -import com.google.api.gax.grpc.FlowControlSettings; -import com.google.cloud.pubsub.spi.v1.Publisher; -import com.google.common.util.concurrent.SettableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.TopicName; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.RecordTooLargeException; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.serialization.Serializer; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Kafka client that publishes records to Google Cloud Pub/Sub. - */ -public class PubsubProducer implements Producer { - - private static final Logger log = LoggerFactory.getLogger(PubsubProducer.class); - private static final long DEFAULT_ELEMENT_COUNT_THRESHOLD = 950L; - - private final String project; - private final String topic; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final long batchSize; - private final int bufferMemory; - private final boolean isAcks; - private final int maxRequestSize; - private final long lingerMs; - private Publisher publisher; - - private AtomicBoolean closed; - - private PubsubProducer(Builder builder) { - project = builder.project; - topic = builder.topic; - batchSize = builder.batchSize; - isAcks = builder.isAcks; - maxRequestSize = builder.maxRequestSize; - keySerializer = builder.keySerializer; - valueSerializer = builder.valueSerializer; - lingerMs = builder.lingerMs; - bufferMemory = builder.bufferMemory; - publisher = newPublisher(); - closed = new AtomicBoolean(false); - } - - public PubsubProducer(Map configs) { - this(new PubsubProducerConfig(configs), null, null); - } - - public PubsubProducer(Map configs, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig( - PubsubProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - public PubsubProducer(Properties properties) { - this(new PubsubProducerConfig(properties), null, null); - } - - public PubsubProducer(Properties properties, Serializer keySerializer, - Serializer valueSerializer) { - this(new PubsubProducerConfig( - PubsubProducerConfig.addSerializerToConfig(properties, keySerializer, valueSerializer)), - keySerializer, valueSerializer); - } - - private PubsubProducer(PubsubProducerConfig configs, Serializer keySerializer, - Serializer valueSerializer) { - try { - log.trace("Starting the Pubsub producer"); - - if (keySerializer == null) { - this.keySerializer = - configs.getConfiguredInstance( - PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.keySerializer.configure(configs.originals(), true); - } else { - configs.ignore(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - - if (valueSerializer == null) { - this.valueSerializer = configs.getConfiguredInstance( - PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); - this.valueSerializer.configure(configs.originals(), false); - } else { - configs.ignore(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - } catch (Exception e) { - throw new RuntimeException(e); - } - batchSize = configs.getLong(PubsubProducerConfig.BATCH_SIZE_CONFIG); - isAcks = configs.getString(PubsubProducerConfig.ACKS_CONFIG).matches("1|all"); - project = configs.getString(PubsubProducerConfig.PROJECT_CONFIG); - topic = configs.getString(PubsubProducerConfig.TOPIC_CONFIG); - maxRequestSize = configs.getInt(PubsubProducerConfig.MAX_REQUEST_SIZE_CONFIG); - lingerMs = configs.getLong(PubsubProducerConfig.LINGER_MS_CONFIG); - bufferMemory = configs.getInt(PubsubProducerConfig.BUFFER_MEMORY_CONFIG); - publisher = newPublisher(); - closed = new AtomicBoolean(false); - log.debug("Producer successfully initialized."); - } - - private Publisher newPublisher() { - TopicName topicName = TopicName.create(project, topic); - Publisher newPub = null; - try { - newPub = Publisher.newBuilder(topicName) - .setBundlingSettings(BundlingSettings.newBuilder() - .setElementCountThreshold(DEFAULT_ELEMENT_COUNT_THRESHOLD) - .setDelayThreshold(Duration.millis(lingerMs)) - .setRequestByteThreshold(batchSize) - .build()) - .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(bufferMemory) - .build()) - .build(); - } catch (IOException e) { - log.error("Exception occurred: " + e); - } - return newPub; - } - - /** - * Sends the given record. - */ - public Future send(ProducerRecord record) { - return send(record, null); - } - - /** - * Sends the given record and invokes the specified callback. - * The given record must have the same topic as the producer. - */ - public Future send(ProducerRecord record, Callback callback) { - log.trace("Received " + record.toString()); - if (closed.get()) { - throw new RuntimeException("Publisher is closed"); - } - - if (!record.topic().equals(topic)) { - throw new IllegalArgumentException("The record's topic must be the same as the Producer's topic."); - } - - Map attributes = new HashMap<>(); - - byte[] serializedKey = ByteString.EMPTY.toByteArray(); - if (record.key() != null) { - serializedKey = this.keySerializer.serialize(topic, record.key()); - attributes - .put("key", new String(serializedKey, StandardCharsets.ISO_8859_1)); - } - - byte[] valueBytes = ByteString.EMPTY.toByteArray(); - if (record.value() != null) { - valueBytes = valueSerializer.serialize(topic, record.value()); - } - - checkRecordSize(Records.LOG_OVERHEAD + Record.recordSize(serializedKey, valueBytes)); - - PubsubMessage message = - PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(valueBytes)) - .putAllAttributes(attributes) - .build(); - RpcFuture messageIdFuture = publisher.publish(message); - Future future = SettableFuture.create(); - RecordMetadata metadata = new RecordMetadata(new TopicPartition(topic, 0), - 0, 0, System.currentTimeMillis(), 0, serializedKey.length, valueBytes.length); - - if (callback != null) { - if (isAcks) { - messageIdFuture.addCallback(new RpcFutureCallback() { - @Override - public void onFailure(Throwable t) { - callback.onCompletion(metadata, new ExecutionException(t)); - } - - @Override - public void onSuccess(String result) { - callback.onCompletion(metadata, null); - } - }); - } else { - callback.onCompletion(metadata, null); - } - } - - return future; - } - - private void checkRecordSize(int size) { - if (size > this.maxRequestSize) { - throw new RecordTooLargeException( - "Message is " + size + " bytes which is larger than max request size you have" - + " configured"); - } - } - - /** - * Flushes records that have accumulated. - */ - public void flush() { - log.debug("Flushing..."); - // Shut down publisher to flush the logs, then immediately restart it. - try { - publisher.shutdown(); - publisher = newPublisher(); - } catch (Exception e) { - log.error("Exception occurred during flush: " + e); - } - } - - /** - * Not supported by this implementation. - */ - public List partitionsFor(String topic) { - throw new NotImplementedException("Partitions not supported"); - } - - /** - * Not supported by this implementation. - */ - public Map metrics() { - throw new NotImplementedException("Metrics not supported."); - } - - /** - * Closes the producer. - */ - public void close() { - close(0, null); - } - - /** - * Closes the producer with the given timeout. - */ - public void close(long timeout, TimeUnit unit) { - if (timeout < 0) { - throw new IllegalArgumentException("Timeout cannot be negative."); - } - if (closed.getAndSet(true)) { - throw new IllegalStateException("Cannot close a producer if already closed."); - } - try { - publisher.shutdown(); - } catch (Exception e) { - log.error("Exception occurred during close: " + e); - } - log.debug("Closed producer"); - } - - /** - * PubsubProducer.Builder is used to create an instance of the publisher, with the specified - * properties and configurations. - */ - public static class Builder { - - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final String topic; - - private long batchSize; - private long lingerMs; - private int bufferMemory; - private boolean isAcks; - private int maxRequestSize; - - public Builder(String project, String topic, Serializer keySerializer, Serializer valueSerializer) { - Preconditions - .checkArgument(project != null && keySerializer != null && valueSerializer != null); - this.project = project; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - this.topic = topic; - setDefaults(); - } - - private void setDefaults() { - // this is where to set 'regular' fields w/o side effects - this.batchSize = PubsubProducerConfig.DEFAULT_BATCH_SIZE; - this.isAcks = PubsubProducerConfig.DEFAULT_ACKS; - this.maxRequestSize = PubsubProducerConfig.DEFAULT_MAX_REQUEST_SIZE; - this.lingerMs = PubsubProducerConfig.DEFAULT_LINGER_MS; - this.bufferMemory = PubsubProducerConfig.DEFAULT_BUFFER_MEMORY; - } - - public Builder batchSize(long val) { - Preconditions.checkArgument(val > 0); - batchSize = val; - return this; - } - - public Builder isAcks(boolean val) { - isAcks = val; - return this; - } - - public Builder maxRequestSize(int val) { - Preconditions.checkArgument(val >= 0); - maxRequestSize = val; - return this; - } - - public Builder lingerMs(long val) { - Preconditions.checkArgument(val >= 0); - lingerMs = val; - return this; - } - - public Builder bufferMemory(int val) { - Preconditions.checkArgument(val >= 0); - bufferMemory = val; - return this; - } - - public PubsubProducer build() { - return new PubsubProducer(this); - } - } - -} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java deleted file mode 100644 index 772ca0fb..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubsubProducerConfig.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.producer; - -import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.common.serialization.Serializer; - -/** - * Provides the configurations for a PubsubProducer instance. - */ -public class PubsubProducerConfig extends AbstractConfig { - private static final ConfigDef CONFIG; - - public static final String KEY_SERIALIZER_CLASS_CONFIG = "key.serializer"; - private static final String KEY_SERIALIZER_CLASS_DOC = "Serializer class for key that implements " - + "the Serializer interface."; - - public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; - private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that " - + "implements the Serializer interface."; - - public static final String BATCH_SIZE_CONFIG = "batch.size"; - private static final String BATCH_SIZE_DOC = "Batch size to use for publishing."; - - public static final String ACKS_CONFIG = "acks"; - private static final String ACKS_DOC = "Whether server acks are needed before a publish " - + "request completes."; - - public static final String PROJECT_CONFIG = "project"; - private static final String PROJECT_DOC = "GCP project to use with the publisher."; - - public static final String TOPIC_CONFIG = "topic"; - private static final String TOPIC_DOC = "Topic to which messages are published."; - - public static final String MAX_REQUEST_SIZE_CONFIG = "max.request.size"; - private static final String MAX_REQUEST_SIZE_DOC = "The maximum size of a request in bytes."; - - public static final String LINGER_MS_CONFIG = "linger.ms"; - private static final String LINGER_MS_DOC = "Referred to as delayThreshold in CPS."; - - public static final String BUFFER_MEMORY_CONFIG = "buffer.memory"; - private static final String BUFFER_MEMORY_DOC = "Referred to as maxOutstandingRequestBytes in CPS."; - - public static final int DEFAULT_BATCH_SIZE = 1; - public static final long DEFAULT_LINGER_MS = 0L; - public static final boolean DEFAULT_ACKS = true; - public static final int DEFAULT_MAX_REQUEST_SIZE = 1024*1024; - public static final int DEFAULT_BUFFER_MEMORY = 32 * 1024 * 1024; - - static { - CONFIG = - new ConfigDef() - .define( - KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) - .define( - VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, - VALUE_SERIALIZER_CLASS_DOC) - .define(BUFFER_MEMORY_CONFIG, Type.LONG, DEFAULT_BUFFER_MEMORY, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(TOPIC_CONFIG, Type.STRING, Importance.HIGH, TOPIC_DOC) - .define(BATCH_SIZE_CONFIG, Type.INT, DEFAULT_BATCH_SIZE, Importance.MEDIUM, - BATCH_SIZE_DOC) - .define(LINGER_MS_CONFIG, Type.LONG, DEFAULT_LINGER_MS, atLeast(0L), Importance.MEDIUM, LINGER_MS_DOC) - .define(ACKS_CONFIG, Type.STRING, "1", Importance.MEDIUM, ACKS_DOC) - .define(MAX_REQUEST_SIZE_CONFIG, Type.INT, DEFAULT_MAX_REQUEST_SIZE, atLeast(0), - Importance.MEDIUM, MAX_REQUEST_SIZE_DOC); - } - - PubsubProducerConfig(Map properties) { - super(CONFIG, properties); - } - - public static Map addSerializerToConfig(Map configs, - Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap(); - newConfigs.putAll(configs); - if (keySerializer != null) { - newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); - } - if (valueSerializer != null) { - newConfigs.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); - } - return newConfigs; - } - - public static Properties addSerializerToConfig(Properties properties, - Serializer keySerializer, Serializer valueSerializer) { - Properties newProperties = new Properties(); - newProperties.putAll(properties); - if (keySerializer != null) { - newProperties.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); - } - if (valueSerializer != null) { - newProperties.put(VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer.getClass()); - } - return newProperties; - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java deleted file mode 100644 index 95d83999..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThread.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.tests; - -import com.google.pubsub.clients.producer.PubsubProducer; -import com.google.pubsub.clients.producer.PubsubProducer.Builder; -import java.util.Properties; -import org.apache.kafka.common.serialization.StringSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Creates a thread that sends a given series of messages. Serves as a sanity check for the - * PubsubProducer implementation. - */ -public class ProducerThread implements Runnable { - private String command; - private PubsubProducer producer; - private String topic; - private static final Logger log = LoggerFactory.getLogger(ProducerThread.class); - private int numMessages; - - public ProducerThread(String s, Properties props, String topic, int numMessages) { - this.command = s; - this.producer = new Builder<>(props.getProperty("project"), topic, new StringSerializer(), new StringSerializer()) - .batchSize(Integer.parseInt(props.getProperty("batch.size"))) - .isAcks(props.getProperty("acks").matches("1|all")) - .lingerMs(Long.parseLong(props.getProperty("linger.ms"))) - .build(); - this.topic = topic; - this.numMessages = numMessages; - } - - public void run() { - log.info("Start running the command"); - processCommand(); - log.info("End running the command"); - } - - private void processCommand() { - try { - for (int i = 0; i < numMessages; i++) { - ProducerRecord msg = new ProducerRecord<>(topic, "hello" + command + i); - producer.send( - msg, - new Callback() { - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - log.error("Exception sending the message: " + exception.getMessage()); - } else { - log.info("Successfully sent message"); - } - } - } - ); - } - Thread.sleep(5000); - producer.close(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - public String toString() { - return command; - } -} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java deleted file mode 100644 index f94b3cc3..00000000 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/tests/ProducerThreadPool.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.tests; - -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Properties; -import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * Example main class to start up several ProducerThreads with a specified topic. - * Number of threads will also be specified by a command-line argument. - * arg[0] = topic, arg[1] = number of threads, arg[2] = number of messages to send. - * Topic is required; if number of threads is not specified, 2 is the default. - * If number of messages is not specified, 1 is the default. - */ -public class ProducerThreadPool { - private static final Logger log = LoggerFactory.getLogger(ProducerThreadPool.class); - - public static void main(String[] args) { - Preconditions.checkArgument(args.length > 0); - ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); - threadFactoryBuilder.setNameFormat("pubsub-producer-thread"); - threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread t, Throwable e) { - log.error(t + " throws exception: " + e); - } - }); - - ExecutorService executor = Executors.newCachedThreadPool(threadFactoryBuilder.build()); - - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("project", "dataproc-kafka-test") - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") - .put("batch.size", "1000") - .put("linger.ms", "1") - .build() - ); - - int numThreads = 2; - if (args.length > 1) { - numThreads = Integer.parseInt(args[1]); - } - int numMessages = 1; - if (args.length > 2) { - numMessages = Integer.parseInt(args[2]); - } - - for (int i = 0; i < numThreads; i++) { - Runnable worker = new ProducerThread("" + i, props, args[0], numMessages); - executor.execute(worker); - } - - executor.shutdown(); - try { - if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { - executor.shutdownNow(); - if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { - log.error("Executor did not terminate"); - } - } - } catch (InterruptedException ie) { - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - } -} diff --git a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java new file mode 100644 index 00000000..2541a1c5 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java @@ -0,0 +1,45 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package org.apache.kafka.clients.producer; + +import java.util.Map; +import java.util.Properties; + +/** + * This is an adapter to control the ProducerConfig class since it's constructors are package-private. + */ +public class ProducerConfigAdapter { + + public static ProducerConfig getConsumerConfig(Properties properties) { + return getConsumerConfig(properties); + } + + public static ProducerConfig getConsumerConfig(Map configurations) { + addDefaultKafkaRequiredConfigs(configurations); + return new ProducerConfig(configurations); + } + + private static void addDefaultKafkaRequiredConfigs(Map map) { + if (!map.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) { + map.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:8080"); + } + + if (!map.containsKey(ProducerConfig.LINGER_MS_CONFIG)) { + map.put(ProducerConfig.LINGER_MS_CONFIG, 1); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java new file mode 100644 index 00000000..56ac78de --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java @@ -0,0 +1,97 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.Assert; +import org.junit.rules.ExpectedException; + +import java.util.Properties; + +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.clients.producer.Config.PubSubProducerConfig; + +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringSerializer; + +public class ConfigTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void successAllConfigsProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("acks", "-1") + .put("project", "unit-test-project") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + Config testConfig = new Config(props); + + Assert.assertEquals("Project config equals unit-test-project.", + "unit-test-project", testConfig.getAdditionalConfigs().getString(PubSubProducerConfig.PROJECT_CONFIG)); + + Assert.assertNotNull( + "Key serializer must not be null.", testConfig.getKafkaConfigs() + .getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + Assert.assertEquals( + "Key serializer config must equal StringSerializer.", StringSerializer.class, + testConfig.getKafkaConfigs().getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)); + + Assert.assertNotNull( + "Value serializer must not be null.", testConfig.getKafkaConfigs(). + getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + Assert.assertEquals( + "Value serializer config must equal StringSerializer.", StringSerializer.class, + testConfig.getKafkaConfigs().getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)); + } + + @Test + public void noSerializerProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("topic", "unit-test-topic") + .put("project", "unit-test-project") + .build() + ); + + exception.expect(ConfigException.class); + + new Config(props); + } + + @Test + public void noTopicProvided() { + Properties props = new Properties(); + props.putAll(new ImmutableMap.Builder<>() + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .build() + ); + + exception.expect(ConfigException.class); + + new Config(props); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java new file mode 100644 index 00000000..39de8019 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java @@ -0,0 +1,317 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import org.junit.Test; +import org.junit.Before; +import org.junit.Assert; +import org.junit.runner.RunWith; + +import org.mockito.Matchers; +import org.mockito.ArgumentCaptor; + +import com.google.pubsub.v1.TopicName; +import com.google.pubsub.v1.PubsubMessage; + +import com.google.cloud.pubsub.v1.Publisher; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.Publisher.Builder; + +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.batching.BatchingSettings; + +import com.google.common.collect.ImmutableMap; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.ProducerInterceptor; + +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; + +import java.io.PrintStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.ByteArrayOutputStream; + +import java.util.Map; +import java.util.List; +import java.util.ArrayList; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import com.google.api.core.ApiFuture; + +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; +import static org.powermock.api.mockito.PowerMockito.when; +import org.powermock.core.classloader.annotations.PrepareForTest; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Publisher.class, TopicAdminClient.class}) +public class KafkaProducerTest { + + private ApiFuture lf; + private Publisher stub; + private Builder stubBuilder; + private Properties properties; + private Serializer serializer; + private Deserializer deserializer; + private TopicAdminClient topicAdmin; + private ArgumentCaptor captor; + private KafkaProducer publisher; + + @Before + public void setUp() throws IOException { + properties = new Properties(); + properties.putAll(new ImmutableMap.Builder<>() + .put("acks", "1") + .put("project", "project") + .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + .put("value.serializer", "org.apache.kafka.common.serialization.IntegerSerializer") + .build() + ); + + lf = PowerMockito.mock(ApiFuture.class); + captor = ArgumentCaptor.forClass(PubsubMessage.class); + + stub = PowerMockito.mock(Publisher.class, Mockito.RETURNS_DEEP_STUBS); + stubBuilder = PowerMockito.mock(Builder.class, Mockito.RETURNS_DEEP_STUBS); + topicAdmin = PowerMockito.mock(TopicAdminClient.class, Mockito.RETURNS_DEEP_STUBS); + + PowerMockito.mockStatic(Publisher.class); + PowerMockito.mockStatic(TopicAdminClient.class); + + when(stubBuilder.build()).thenReturn(stub); + when(stub.publish(captor.capture())).thenReturn(lf); + when(TopicAdminClient.create()).thenReturn(topicAdmin); + when(Publisher.defaultBuilder(Matchers.any())).thenReturn(stubBuilder); + when(stubBuilder.setRetrySettings(Matchers.any())).thenReturn(stubBuilder); + when(stubBuilder.setBatchingSettings(Matchers.any())).thenReturn(stubBuilder); + + publisher = new KafkaProducer(properties, null, null); + } + + @Test + public void flush() { + publisher.send(new ProducerRecord("topic", 123)); + + publisher.flush(); + + publisher.send(new ProducerRecord("topic", 456)); + + try { + Mockito.verify(stub, Mockito.times(1)).shutdown(); + + Mockito.verify(stub, Mockito.times(2)) + .publish(Matchers.any()); + + Mockito.verify(stubBuilder, Mockito.times(2)).build(); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + @Test + public void callback() { + serializer = new IntegerSerializer(); + + Callback cb = new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + Assert.assertEquals(metadata.topic(), "topic"); + + Assert.assertEquals(metadata.serializedKeySize(), 0); + + Assert.assertEquals(metadata.serializedValueSize(), + serializer.serialize("topic", 123).length); + } + }; + + publisher.send(new ProducerRecord("topic", 123), cb); + } + + @Test + public void serializers() { + publisher.send(new ProducerRecord("topic", 123)); + + deserializer = new StringDeserializer(); + + Assert.assertEquals("Key should be an empty string.", + "", deserializer.deserialize("topic", + captor.getValue().getAttributesMap().get("key").getBytes())); + + deserializer = new IntegerDeserializer(); + + Assert.assertEquals("Value should be the one previously provided.", + 123, deserializer.deserialize("topic", + captor.getValue().getData().toByteArray())); + } + + @Test (expected = NullPointerException.class) + public void publishNull() { + publisher.send(new ProducerRecord("topic", null)); + } + + @Test (expected = IllegalArgumentException.class) + public void negativeTimeout() { + publisher.close(-1, TimeUnit.SECONDS); + } + + @Test + public void closeOnCompletion() { + publisher.send(new ProducerRecord("topic", 123)); + + publisher.close(); + + try { + publisher.send(new ProducerRecord("topic", 123)); + } catch (Exception e) {} + + try { + Mockito.verify(stub, Mockito.times(1)).shutdown(); + + Mockito.verify(stub, Mockito.times(1)) + .publish(Matchers.any()); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + @Test (expected = NullPointerException.class) + public void emptyRecordPublish() { + publisher.send(null); + } + + @Test (expected = NullPointerException.class) + public void publishEmptyMessage() { + KafkaProducer pub = new KafkaProducer(properties, + null, new StringSerializer()); + pub.send(new ProducerRecord("topic", "")); + } + + @Test + public void numberOfPublishIssued() { + publisher.send(new ProducerRecord("topic", 123)); + + Mockito.verify(stub, Mockito.times(1)) + .publish(Matchers.any()); + + publisher.send(new ProducerRecord("topic", 456)); + + Mockito.verify(stub, Mockito.times(2)) + .publish(Matchers.any()); + } + + @Test (expected = RuntimeException.class) + public void publishToClosedPublisher() { + publisher.close(); + + publisher.send(new ProducerRecord("topic", 123)); + } + + @Test + public void interceptRecords() { + int key = 123; + + PrintStream original = System.out; + OutputStream os = new ByteArrayOutputStream(100); + + System.setOut(new PrintStream(os)); + + properties.put("interceptor.classes", MultiplyByTenInterceptor.class.getName()); + + KafkaProducer pub = + new KafkaProducer(properties, null, null); + + pub.send(new ProducerRecord("topic", key)); + + System.setOut(original); + + Assert.assertEquals(10 * key, Integer.parseInt(os.toString())); + } + + //This one is supposed to work as the previous but log an exception. + @Test + public void interceptRecordsWithException() { + int key = 123; + + PrintStream original = System.out; + OutputStream os = new ByteArrayOutputStream(); + + System.setOut(new PrintStream(os)); + + List list = new ArrayList<>(); + list.add(MultiplyByTenInterceptor.class.getName()); + list.add(ThrowExceptionInterceptor.class.getName()); + + properties.put("interceptor.classes", list); + + KafkaProducer pub = + new KafkaProducer(properties, null, null); + + pub.send(new ProducerRecord("topic", key)); + + System.setOut(original); + + Assert.assertEquals(10 * key, Integer.parseInt(os.toString())); + } + + public static class MultiplyByTenInterceptor implements ProducerInterceptor { + @Override + public ProducerRecord onSend(ProducerRecord producerRecord) { + int updatedValue = 10 * producerRecord.value(); + System.out.print(updatedValue); + return new ProducerRecord(producerRecord.topic(), producerRecord.key(), updatedValue); + } + + @Override + public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } + + @Override + public void close() { + System.out.print("Closed"); + } + + @Override + public void configure(Map map) { } + } + + public static class ThrowExceptionInterceptor implements ProducerInterceptor { + @Override + public ProducerRecord onSend(ProducerRecord producerRecord) { + throw new RuntimeException(); + } + + @Override + public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } + + @Override + public void close() { + System.out.print("Closed"); + } + + @Override + public void configure(Map map) { } + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java deleted file mode 100644 index 4baefb35..00000000 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/PubsubProducerConfigTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.google.pubsub.clients.producer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -import com.google.common.collect.ImmutableMap; -import java.util.Properties; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.serialization.Serializer; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - - -public class PubsubProducerConfigTest { - - @Rule - public final ExpectedException exception = ExpectedException.none(); - - @Test - public void testSuccessAllConfigsProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("project", "unit-test-project") - .put("topic", "unit-test-topic") - .build() - ); - - PubsubProducerConfig testConfig = new PubsubProducerConfig(props); - - assertEquals("Project config equals unit-test-project.", "unit-test-project", testConfig.getString(PubsubProducerConfig.PROJECT_CONFIG)); - assertNotNull("Key serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); - assertNotNull("Value serializer must not be null.", testConfig.getConfiguredInstance(PubsubProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); - } - - @Test - public void testNoSerializerProvided() { - Properties props = new Properties(); - props.put(PubsubProducerConfig.PROJECT_CONFIG, "unit-test-project"); - - exception.expect(ConfigException.class); - new PubsubProducerConfig(props); - - } - - @Test - public void testNoProjectProvided() { - Properties props = new Properties(); - props.putAll(new ImmutableMap.Builder<>() - .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .build() - ); - - exception.expect(ConfigException.class); - new PubsubProducerConfig(props); - } -} From be3492ae3f8e70fa240ccf5cc395d16bb9d7ebdd Mon Sep 17 00:00:00 2001 From: pietrzykp Date: Fri, 22 Sep 2017 17:30:55 -0400 Subject: [PATCH 134/140] Mapped api ack (#122) Commit functionality: * Copy & modify GCP Pub/Sub Client Subscriber part on acknowledging and extending deadlines * Manual and auto commit * Sync and async commit --- pubsub-mapped-api/pom.xml | 20 +- .../pubsub/clients/consumer/Config.java | 48 +- .../clients/consumer/KafkaConsumer.java | 157 ++--- .../consumer/PubSubConsumerConfig.java | 35 +- .../consumer/ack/AckReplyConsumer.java | 34 + .../ack/MappedApiMessageReceiver.java | 11 + .../consumer/ack/MessageDispatcher.java | 624 ++++++++++++++++++ .../clients/consumer/ack/MessageReceiver.java | 48 ++ .../clients/consumer/ack/MessageWaiter.java | 67 ++ .../ack/PollingSubscriberConnection.java | 165 +++++ .../clients/consumer/ack/StatusUtil.java | 49 ++ .../clients/consumer/ack/Subscriber.java | 374 +++++++++++ .../src/main/resources/log4j.properties | 2 +- .../clients/consumer/KafkaConsumerTest.java | 21 +- .../clients/consumer/ack/FakeClock.java | 42 ++ .../ack/FakeScheduledExecutorService.java | 353 ++++++++++ .../ack/FakeSubscriberServiceImpl.java | 180 +++++ .../clients/consumer/ack/SubscriberTest.java | 475 +++++++++++++ .../clients/producer/KafkaProducerTest.java | 64 +- 19 files changed, 2632 insertions(+), 137 deletions(-) create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/AckReplyConsumer.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MappedApiMessageReceiver.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageReceiver.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageWaiter.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/StatusUtil.java create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeClock.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeScheduledExecutorService.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeSubscriberServiceImpl.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index faa6ba4a..5c99f4fd 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -19,6 +19,9 @@ MappedApi http://maven.apache.org + + 1.3.0 + @@ -28,27 +31,27 @@ io.grpc grpc-netty - 1.3.0 + ${grpc.version} io.grpc grpc-auth - 1.3.0 + ${grpc.version} io.grpc grpc-protobuf - 1.3.0 + ${grpc.version} io.grpc grpc-stub - 1.3.0 + ${grpc.version} io.grpc grpc-testing - 1.3.0 + ${grpc.version} test @@ -78,7 +81,7 @@ org.apache.kafka - kafka-clients + kafka_2.10 0.10.1.1 @@ -138,5 +141,10 @@ log4j 1.2.17 + + com.google.guava + guava + 23.0 + diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java index ff95fcd4..23bd2def 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java @@ -25,12 +25,19 @@ class Config { private final Boolean allowSubscriptionCreation; private final Boolean allowSubscriptionDeletion; - private final String groupId; private final int maxPollRecords; private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; + private final Boolean enableAutoCommit; + private final Integer autoCommitIntervalMs; + private final Long retryBackoffMs; + private final Integer maxPerRequestChanges; + private final Integer createdSubscriptionDeadlineSeconds; + private final Integer requestTimeoutMs; + private final Integer maxAckExtensionPeriod; + Config(Map configs) { this(ConsumerConfigCreator.getConsumerConfig(configs), @@ -81,11 +88,21 @@ private Config(ConsumerConfig consumerConfig, //this is a limit on each poll for each topic this.maxPollRecords = consumerConfig.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); + this.enableAutoCommit = consumerConfig.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + this.autoCommitIntervalMs = consumerConfig.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); + this.retryBackoffMs = consumerConfig.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + this.requestTimeoutMs = consumerConfig.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + //PubSub-specific options this.allowSubscriptionCreation = pubSubConsumerConfig.getBoolean(PubSubConsumerConfig.SUBSCRIPTION_ALLOW_CREATE_CONFIG); this.allowSubscriptionDeletion = pubSubConsumerConfig.getBoolean(PubSubConsumerConfig.SUBSCRIPTION_ALLOW_DELETE_CONFIG); + this.maxPerRequestChanges = pubSubConsumerConfig.getInt(PubSubConsumerConfig.MAX_PER_REQUEST_CHANGES_CONFIG); + this.createdSubscriptionDeadlineSeconds = pubSubConsumerConfig.getInt( + PubSubConsumerConfig.CREATED_SUBSCRIPTION_DEADLINE_SECONDS_CONFIG); + this.maxAckExtensionPeriod = pubSubConsumerConfig.getInt(PubSubConsumerConfig.MAX_ACK_EXTENSION_PERIOD_SECONDS_CONFIG); + Preconditions.checkNotNull(this.allowSubscriptionCreation); Preconditions.checkNotNull(this.allowSubscriptionDeletion); @@ -117,6 +134,34 @@ Deserializer getValueDeserializer() { return valueDeserializer; } + Boolean getEnableAutoCommit() { + return enableAutoCommit; + } + + Integer getAutoCommitIntervalMs() { + return autoCommitIntervalMs; + } + + Long getRetryBackoffMs() { + return retryBackoffMs; + } + + Integer getMaxPerRequestChanges() { + return maxPerRequestChanges; + } + + Integer getRequestTimeoutMs() { + return requestTimeoutMs; + } + + Integer getCreatedSubscriptionDeadlineSeconds() { + return createdSubscriptionDeadlineSeconds; + } + + Integer getMaxAckExtensionPeriod() { + return maxAckExtensionPeriod; + } + private Deserializer handleDeserializer(ConsumerConfig configs, String configString, Deserializer providedDeserializer, boolean isKey) { Deserializer deserializer; @@ -129,5 +174,4 @@ private Deserializer handleDeserializer(ConsumerConfig configs, String configStr } return deserializer; } - } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index 7e3f371a..b0096c0d 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -16,6 +16,7 @@ package com.google.pubsub.clients.consumer; import com.google.api.client.util.Base64; +import com.google.api.gax.batching.FlowControlSettings; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -23,8 +24,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.Empty; +import com.google.pubsub.clients.consumer.ack.MappedApiMessageReceiver; +import com.google.pubsub.clients.consumer.ack.Subscriber; import com.google.pubsub.common.ChannelUtil; -import com.google.pubsub.v1.AcknowledgeRequest; import com.google.pubsub.v1.DeleteSubscriptionRequest; import com.google.pubsub.v1.GetSubscriptionRequest; import com.google.pubsub.v1.ListTopicsRequest; @@ -32,7 +34,6 @@ import com.google.pubsub.v1.PublisherGrpc; import com.google.pubsub.v1.PublisherGrpc.PublisherFutureStub; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.PullRequest; import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; import com.google.pubsub.v1.SubscriberGrpc; @@ -43,18 +44,17 @@ import io.grpc.Channel; import io.grpc.Status.Code; import io.grpc.StatusRuntimeException; -import java.sql.Timestamp; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -98,9 +98,10 @@ public class KafkaConsumer implements Consumer { private final SubscriberFutureStub subscriberFutureStub; private final PublisherFutureStub publisherFutureStub; - private ImmutableMap topicNameToSubscription = ImmutableMap.of(); private ImmutableList topicNames = ImmutableList.of(); + private ImmutableMap topicNameToSubscriber = ImmutableMap.of(); + private int currentPoolIndex; public KafkaConsumer(Map configs) { @@ -146,7 +147,6 @@ private KafkaConsumer(Config configOptions) { this.subscriberFutureStub = subscriberFutureStub; this.publisherFutureStub = publisherFutureStub; - //Kafka-specific options this.config = config; log.debug("PubSub subscriber created"); @@ -162,7 +162,7 @@ public Set assignment() { @Override public Set subscription() { - return topicNameToSubscription.keySet(); + return topicNameToSubscriber.keySet(); } /** @@ -178,14 +178,37 @@ public void subscribe(Collection topics, ConsumerRebalanceListener liste unsubscribe(); List> futureSubscriptions = deputePubsubSubscribesGet(topics); Map subscriptionMap = getSubscriptionsFromPubsub(futureSubscriptions); + Map tempSubscribersMap = new HashMap<>(); + + for(Map.Entry entry: subscriptionMap.entrySet()) { + Subscriber subscriber = getSubscriberFromConfigs(entry); + + tempSubscribersMap.put(entry.getKey(), subscriber); + subscriber.startAsync().awaitRunning(); + } - topicNameToSubscription = ImmutableMap.copyOf(subscriptionMap); + topicNameToSubscriber = ImmutableMap.copyOf(tempSubscribersMap); topicNames = ImmutableList.copyOf(subscriptionMap.keySet()); currentPoolIndex = 0; log.debug("Subscribed to topic(s): {}", Utils.join(topics, ", ")); } + private Subscriber getSubscriberFromConfigs(Entry entry) { + return Subscriber.defaultBuilder(entry.getValue(), + new MappedApiMessageReceiver()) + .setFlowControlSettings(FlowControlSettings.getDefaultInstance()) + .setAutoCommit(config.getEnableAutoCommit()) + .setAutoCommitIntervalMs(config.getAutoCommitIntervalMs()) + .setMaxPullRecords((long) config.getMaxPollRecords()) + .setSubscriberFutureStub(this.subscriberFutureStub) + .setRetryBackoffMs(config.getRetryBackoffMs()) + .setMaxAckExtensionPeriod(config.getMaxAckExtensionPeriod()) + .setMaxPerRequestChanges(config.getMaxPerRequestChanges()) + .setAckRequestTimeoutMs(config.getRequestTimeoutMs()) + .build(); + } + private List> deputePubsubSubscribesGet(Collection topics) { List> responseDatas = new ArrayList<>(); Set usedNames = new HashSet<>(); @@ -259,32 +282,13 @@ private boolean shouldTryToCreateSubscription(ExecutionException e) { && ((StatusRuntimeException)e.getCause()).getStatus().getCode().equals(Code.NOT_FOUND); } - private List> deputePubsubSubscribes(Collection topics) { - Timestamp timestamp = new Timestamp(System.currentTimeMillis()); - - List> responseDatas = new ArrayList<>(); - Set usedNames = new HashSet<>(); - - for (String topic: topics) { - if (!usedNames.contains(topic)) { - String subscriptionString = SUBSCRIPTION_PREFIX + topic + "_" + timestamp.getTime() - + "_" + UUID.randomUUID(); - ListenableFuture deputedSubscription = - deputeSinglePubsubSubscription(subscriptionString, topic); - - responseDatas.add(new ResponseData<>(topic, subscriptionString, deputedSubscription)); - usedNames.add(topic); - } - } - return responseDatas; - } - private ListenableFuture deputeSinglePubsubSubscription(String subscriptionString, String topicName) { return subscriberFutureStub .createSubscription(Subscription.newBuilder() .setName(subscriptionString) .setTopic(TOPIC_PREFIX + topicName) + .setAckDeadlineSeconds(config.getCreatedSubscriptionDeadlineSeconds()) .build()); } @@ -348,12 +352,26 @@ private void checkPatternSubscribePreconditions(Pattern pattern) { @Override public void unsubscribe() { - deleteSubscriptionsIfAllowed(topicNameToSubscription.values()); - topicNameToSubscription = ImmutableMap.of(); + for(Subscriber s: topicNameToSubscriber.values()) { + s.stopAsync().awaitTerminated(); + } + + List currentSubscriptions = getSubscriptionsFromSubcribers(); + deleteSubscriptionsIfAllowed(currentSubscriptions); + + topicNameToSubscriber = ImmutableMap.of(); topicNames = ImmutableList.of(); currentPoolIndex = 0; } + private List getSubscriptionsFromSubcribers() { + List subscriptions = new ArrayList<>(topicNameToSubscriber.size()); + for(Subscriber s: topicNameToSubscriber.values()) { + subscriptions.add(s.getSubscription()); + } + return subscriptions; + } + private void deleteSubscriptionsIfAllowed(Collection subscriptions) { if(!config.getAllowSubscriptionDeletion()) return; @@ -377,96 +395,57 @@ public ConsumerRecords poll(long timeout) { int startedAtIndex = this.currentPoolIndex; try { do { - ResponseData pollData = getPullResponseResponseData(timeout); - PullResponse pullResponse = pollData.getRequestListenableFuture().get(); - List> subscriptionRecords = mapToConsumerRecords(pollData, pullResponse); + String topicName = topicNames.get(this.currentPoolIndex % topicNameToSubscriber.size()); + Subscriber subscriber = topicNameToSubscriber.get(topicName); - this.currentPoolIndex = (this.currentPoolIndex + 1) % topicNameToSubscription.size(); + PullResponse pullResponse = subscriber.pull(timeout); - if (!subscriptionRecords.isEmpty()) { - AcknowledgeRequest acknowledgeRequest = getAcknowledgeRequest(pollData.getSubscriptionFullName(), - pullResponse); - //TODO depute acknowledge message with timeout rather than ack immediately - acknowledgeMessage(acknowledgeRequest); + List> subscriptionRecords = mapToConsumerRecords(topicName, pullResponse); + this.currentPoolIndex = (this.currentPoolIndex + 1) % topicNameToSubscriber.size(); - return getConsumerRecords(pollData, subscriptionRecords); + if (!pullResponse.getReceivedMessagesList().isEmpty()) { + return getConsumerRecords(topicName, subscriptionRecords); } } while (this.currentPoolIndex != startedAtIndex); } catch (InterruptedException e) { throw new InterruptException(e); - } catch (ExecutionException e) { + } catch (ExecutionException | IOException e) { throw new KafkaException(e); } return new ConsumerRecords<>(new HashMap<>()); } - private ConsumerRecords getConsumerRecords(ResponseData pollData, + private ConsumerRecords getConsumerRecords(String topicName, List> subscriptionRecords) { Map>> pollRecords = new HashMap<>(); - TopicPartition topicPartition = new TopicPartition(pollData.getTopicName(), DEFAULT_PARTITION); + TopicPartition topicPartition = new TopicPartition(topicName, DEFAULT_PARTITION); pollRecords.put(topicPartition, subscriptionRecords); return new ConsumerRecords<>(pollRecords); } - private ResponseData getPullResponseResponseData(long timeout) { - String topicName = topicNames.get(this.currentPoolIndex % topicNameToSubscription.size()); - - Subscription subscription = topicNameToSubscription.get(topicName); - ListenableFuture deputedPull = deputeSinglePubsubPull(subscription, timeout); - return new ResponseData<>(topicName, subscription.getName(), deputedPull); - } - - private AcknowledgeRequest getAcknowledgeRequest(String subscription, PullResponse pulled) { - List ackIds = new ArrayList<>(); - for (ReceivedMessage receivedMessage : pulled.getReceivedMessagesList()) { - ackIds.add(receivedMessage.getAckId()); - } - - return AcknowledgeRequest.newBuilder() - .addAllAckIds(ackIds) - .setSubscription(subscription).build(); - } - - private ListenableFuture deputeSinglePubsubPull(Subscription s, long timeout) { - SubscriberFutureStub deadlineFutureStub = - subscriberFutureStub.withDeadlineAfter(timeout, TimeUnit.MILLISECONDS); - - return deadlineFutureStub.pull(PullRequest.newBuilder() - .setSubscription(s.getName()) - .setMaxMessages(config.getMaxPollRecords()) - .setReturnImmediately(true).build()); - } - - private List> mapToConsumerRecords(ResponseData pollData, - PullResponse pulled) { + private List> mapToConsumerRecords(String topicName, PullResponse pulled) { List> subscriptionRecords = new ArrayList<>(); for (ReceivedMessage receivedMessage : pulled.getReceivedMessagesList()) { ConsumerRecord record = prepareKafkaRecord(receivedMessage, - pollData.getTopicName()); + topicName); subscriptionRecords.add(record); } return subscriptionRecords; } - private void acknowledgeMessage(AcknowledgeRequest acknowledgeRequest) - throws ExecutionException, InterruptedException { - ListenableFuture acknowledgeFuture = subscriberFutureStub.acknowledge(acknowledgeRequest); - acknowledgeFuture.get(); - } - private void checkPollPreconditions(long timeout) { Preconditions.checkArgument(timeout >= 0, "Timeout must not be negative"); - if (topicNameToSubscription.isEmpty()) { + if (topicNameToSubscriber.isEmpty()) { throw new IllegalStateException("Consumer is not subscribed to any topics"); } } @@ -483,7 +462,7 @@ private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, //key of Kafka-style message is stored in PubSub attributes (null possible) String key = message.getAttributesOrDefault(KEY_ATTRIBUTE, null); - byte [] deserializedKeyBytes = Base64.decodeBase64(key.getBytes()); + byte [] deserializedKeyBytes = key != null ? Base64.decodeBase64(key.getBytes()) : null; //lengths of serialized value and serialized key int serializedValueSize = message.getData().toByteArray().length; @@ -504,7 +483,7 @@ public void assign(Collection partitions) { @Override public void commitSync() { - throw new UnsupportedOperationException("Not yet implemented"); + commit(true); } @Override @@ -514,7 +493,7 @@ public void commitSync(final Map offsets) { @Override public void commitAsync() { - throw new UnsupportedOperationException("Not yet implemented"); + commit(false); } @Override @@ -528,6 +507,12 @@ public void commitAsync(final Map offsets, throw new UnsupportedOperationException("Not yet implemented"); } + private void commit(boolean sync) { + for (Map.Entry entry : topicNameToSubscriber.entrySet()) { + entry.getValue().commit(sync); + } + } + @Override public void seek(TopicPartition partition, long offset) { throw new UnsupportedOperationException("Not yet implemented"); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java index 0d42f236..88ea8882 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java @@ -14,13 +14,13 @@ package com.google.pubsub.clients.consumer; -import java.util.Map; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; import org.apache.kafka.common.config.ConfigDef.Type; -//TODO Use builder pattern using Kafka's ConsumerConfig class +import java.util.Map; + /** * The consumer configuration keys */ @@ -37,6 +37,20 @@ public class PubSubConsumerConfig extends AbstractConfig { private static final String SUBSCRIPTION_ALLOW_DELETE_DOC = "Determines if subscriptions for non-existing groups should be created"; + /** max.per.request.changes */ + public static final String MAX_PER_REQUEST_CHANGES_CONFIG = "max.per.request.changes"; + private static final String MAX_PER_REQUEST_CHANGES_DOC = + "Maximum of request changes sent in single message (acknowledge & extend deadline)"; + + public static final String CREATED_SUBSCRIPTION_DEADLINE_SECONDS_CONFIG = "created.subscription.deadline.sec"; + private static final String CREATED_SUBSCRIPTION_DEADLINE_SECONDS_DOC = + "If creation of subscriptions is configured, this is the acknowledge deadline they are going to get"; + + + public static final String MAX_ACK_EXTENSION_PERIOD_SECONDS_CONFIG = "max.ack.extension.period.sec"; + private static final String MAX_ACK_EXTENSION_PERIOD_SECONDS_DOC = + "The maximum period a message ack deadline will be extended"; + private static synchronized ConfigDef getInstance() { return new ConfigDef() .define(SUBSCRIPTION_ALLOW_CREATE_CONFIG, @@ -48,7 +62,22 @@ private static synchronized ConfigDef getInstance() { Type.BOOLEAN, false, Importance.MEDIUM, - SUBSCRIPTION_ALLOW_DELETE_DOC); + SUBSCRIPTION_ALLOW_DELETE_DOC) + .define(MAX_PER_REQUEST_CHANGES_CONFIG, + Type.INT, + 1000, + Importance.MEDIUM, + MAX_PER_REQUEST_CHANGES_DOC) + .define(CREATED_SUBSCRIPTION_DEADLINE_SECONDS_CONFIG, + Type.INT, + 20, + Importance.MEDIUM, + CREATED_SUBSCRIPTION_DEADLINE_SECONDS_DOC) + .define(MAX_ACK_EXTENSION_PERIOD_SECONDS_CONFIG, + Type.INT, + 3600, // 60 minutes + Importance.MEDIUM, + MAX_ACK_EXTENSION_PERIOD_SECONDS_DOC); } PubSubConsumerConfig(Map props) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/AckReplyConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/AckReplyConsumer.java new file mode 100644 index 00000000..c2f7f846 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/AckReplyConsumer.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +/** + * Accepts a reply, sending it to the service. + */ +public interface AckReplyConsumer { + /** + * Acknowledges that the message has been successfully processed. The service will not send the + * message again. + */ + void ack(); + + /** + * Signals that the message has not been successfully processed. The service should resend the + * message. + */ + void nack(); +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MappedApiMessageReceiver.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MappedApiMessageReceiver.java new file mode 100644 index 00000000..a8569957 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MappedApiMessageReceiver.java @@ -0,0 +1,11 @@ +package com.google.pubsub.clients.consumer.ack; + +import com.google.pubsub.v1.PubsubMessage; + +public class MappedApiMessageReceiver implements MessageReceiver { + + @Override + public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) { + consumer.ack(); + } +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java new file mode 100644 index 00000000..c55ca83b --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java @@ -0,0 +1,624 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.api.core.ApiClock; +import com.google.api.gax.batching.FlowController; +import com.google.api.gax.batching.FlowController.FlowControlException; +import com.google.api.gax.core.Distribution; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.Empty; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.ReceivedMessage; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import org.apache.kafka.common.KafkaException; +import org.threeten.bp.Duration; +import org.threeten.bp.Instant; + +/** + * Dispatches messages to a message receiver while handling the messages acking and lease + * extensions. + */ +class MessageDispatcher { + private static final Logger logger = Logger.getLogger(MessageDispatcher.class.getName()); + + private static final int INITIAL_ACK_DEADLINE_EXTENSION_SECONDS = 2; + private static final int MAX_ACK_DEADLINE_EXTENSION_SECS = 10 * 60; // 10m + + private final ScheduledExecutorService systemExecutor; + private final ApiClock clock; + + private final Duration ackExpirationPadding; + private final Duration maxAckExtensionPeriod; + private final MessageReceiver receiver; + private final AckProcessor ackProcessor; + + private final FlowController flowController; + private final MessageWaiter messagesWaiter; + + private final PriorityQueue outstandingAckHandlers; + private final HashMap pendingAcks; + private final Set pendingNacks; + + private final Lock alarmsLock; + private final Long retryBackoffMs; + private int messageDeadlineSeconds; + private ScheduledFuture ackDeadlineExtensionAlarm; + private Instant nextAckDeadlineExtensionAlarmTime; + private ScheduledFuture pendingAcksAlarm; + + private final Deque outstandingMessageBatches; + + // To keep track of number of seconds the receiver takes to process messages. + private final Distribution ackLatencyDistribution; + + // ExtensionJob represents a group of {@code AckHandler}s that shares the same expiration. + // + // It is Comparable so that it may be put in a PriorityQueue. + // For efficiency, it is also mutable, so great care should be taken to make sure + // it is not modified while inside the queue. + // The hashcode and equals methods are explicitly not implemented to discourage + // the use of this class as keys in maps or similar containers. + private class ExtensionJob implements Comparable { + Instant creation; + Instant expiration; + int nextExtensionSeconds; + ArrayList ackHandlers; + + ExtensionJob( + Instant creation, + Instant expiration, + int initialAckDeadlineExtension, + ArrayList ackHandlers) { + this.creation = creation; + this.expiration = expiration; + nextExtensionSeconds = initialAckDeadlineExtension; + this.ackHandlers = ackHandlers; + } + + void extendExpiration(Instant now) { + Instant possibleExtension = now.plus(Duration.ofSeconds(nextExtensionSeconds)); + Instant maxExtension = creation.plus(maxAckExtensionPeriod); + expiration = possibleExtension.isBefore(maxExtension) ? possibleExtension : maxExtension; + nextExtensionSeconds = Math.min(2 * nextExtensionSeconds, MAX_ACK_DEADLINE_EXTENSION_SECS); + } + + @Override + public int compareTo(ExtensionJob other) { + return expiration.compareTo(other.expiration); + } + + @Override + public String toString() { + ArrayList ackIds = new ArrayList<>(); + for (AckHandler ah : ackHandlers) { + ackIds.add(ah.ackId); + } + return String.format( + "ExtensionJob {expiration: %s, nextExtensionSeconds: %d, ackIds: %s}", + expiration, nextExtensionSeconds, ackIds); + } + } + + /** Stores the data needed to asynchronously modify acknowledgement deadlines. */ + static class PendingModifyAckDeadline { + final List ackIds; + final int deadlineExtensionSeconds; + + PendingModifyAckDeadline(int deadlineExtensionSeconds, String... ackIds) { + this.ackIds = new ArrayList<>(); + this.deadlineExtensionSeconds = deadlineExtensionSeconds; + for (String ackId : ackIds) { + addAckId(ackId); + } + } + + void addAckId(String ackId) { + ackIds.add(ackId); + } + + @Override + public String toString() { + return String.format( + "PendingModifyAckDeadline{extension: %d sec, ackIds: %s}", + deadlineExtensionSeconds, ackIds); + } + } + + /** Internal representation of a reply to a Pubsub message, to be sent back to the service. */ + public enum AckReply { + ACK, + NACK + } + + /** + * Handles callbacks for acking/nacking messages from the {@link + * MessageReceiver}. + */ + private class AckHandler implements FutureCallback { + private final String ackId; + private final int outstandingBytes; + private final AtomicBoolean acked; + private final long receivedTimeMillis; + + AckHandler(String ackId, int outstandingBytes) { + this.ackId = ackId; + this.outstandingBytes = outstandingBytes; + acked = new AtomicBoolean(false); + receivedTimeMillis = clock.millisTime(); + } + + @Override + public void onFailure(Throwable t) { + logger.log( + Level.WARNING, + "MessageReceiver failed to processes ack ID: " + ackId + ", the message will be nacked.", + t); + acked.getAndSet(true); + synchronized (pendingNacks) { + pendingNacks.add(ackId); + } + flowController.release(1, outstandingBytes); + messagesWaiter.incrementPendingMessages(-1); + processOutstandingBatches(); + } + + @Override + public void onSuccess(AckReply reply) { + switch (reply) { + case ACK: + synchronized (pendingAcks) { + pendingAcks.put(ackId, this); + } + // Record the latency rounded to the next closest integer. + ackLatencyDistribution.record( + Ints.saturatedCast( + (long) Math.ceil((clock.millisTime() - receivedTimeMillis) / 1000D))); + break; + case NACK: + synchronized (pendingNacks) { + pendingNacks.add(ackId); + } + break; + default: + throw new IllegalArgumentException(String.format("AckReply: %s not supported", reply)); + } + flowController.release(1, outstandingBytes); + messagesWaiter.incrementPendingMessages(-1); + processOutstandingBatches(); + } + } + + public interface AckProcessor { + ListenableFuture> sendAckOperations( + List acksToSend, List ackDeadlineExtensions); + } + + MessageDispatcher( + MessageReceiver receiver, + AckProcessor ackProcessor, + Duration ackExpirationPadding, + Duration maxAckExtensionPeriod, + Distribution ackLatencyDistribution, + FlowController flowController, + ScheduledExecutorService systemExecutor, + ApiClock clock, + Long retryBackoffMs) { + this.systemExecutor = systemExecutor; + this.ackExpirationPadding = ackExpirationPadding; + this.maxAckExtensionPeriod = maxAckExtensionPeriod; + this.receiver = receiver; + this.ackProcessor = ackProcessor; + this.flowController = flowController; + this.retryBackoffMs = retryBackoffMs; + outstandingMessageBatches = new LinkedList<>(); + outstandingAckHandlers = new PriorityQueue<>(); + pendingAcks = new HashMap<>(); + pendingNacks = new HashSet<>(); + // 601 buckets of 1s resolution from 0s to MAX_ACK_DEADLINE_SECONDS + this.ackLatencyDistribution = ackLatencyDistribution; + alarmsLock = new ReentrantLock(); + nextAckDeadlineExtensionAlarmTime = Instant.ofEpochMilli(Long.MAX_VALUE); + messagesWaiter = new MessageWaiter(); + this.clock = clock; + } + + void stop() { + messagesWaiter.waitNoMessages(); + alarmsLock.lock(); + try { + if (ackDeadlineExtensionAlarm != null) { + ackDeadlineExtensionAlarm.cancel(true); + ackDeadlineExtensionAlarm = null; + } + } finally { + alarmsLock.unlock(); + } + + extendAckDeadlines(new ArrayList<>()); + } + + void setMessageDeadlineSeconds(int messageDeadlineSeconds) { + this.messageDeadlineSeconds = messageDeadlineSeconds; + } + + static class OutstandingMessagesBatch { + private final Deque messages; + private final Runnable doneCallback; + + static class OutstandingMessage { + private final ReceivedMessage receivedMessage; + private final AckHandler ackHandler; + + OutstandingMessage(ReceivedMessage receivedMessage, AckHandler ackHandler) { + this.receivedMessage = receivedMessage; + this.ackHandler = ackHandler; + } + + ReceivedMessage receivedMessage() { + return receivedMessage; + } + + AckHandler ackHandler() { + return ackHandler; + } + } + + OutstandingMessagesBatch(Runnable doneCallback) { + this.messages = new LinkedList<>(); + this.doneCallback = doneCallback; + } + + void addMessage(ReceivedMessage receivedMessage, AckHandler ackHandler) { + this.messages.add(new OutstandingMessage(receivedMessage, ackHandler)); + } + + public Deque messages() { + return messages; + } + } + + void processReceivedMessages(List messages, Runnable doneCallback) { + if (messages.isEmpty()) { + doneCallback.run(); + return; + } + messagesWaiter.incrementPendingMessages(messages.size()); + + OutstandingMessagesBatch outstandingBatch = new OutstandingMessagesBatch(doneCallback); + final ArrayList ackHandlers = new ArrayList<>(messages.size()); + for (ReceivedMessage message : messages) { + AckHandler ackHandler = + new AckHandler(message.getAckId(), message.getMessage().getSerializedSize()); + ackHandlers.add(ackHandler); + outstandingBatch.addMessage(message, ackHandler); + } + + Instant expiration = Instant.ofEpochMilli(clock.millisTime()) + .plusSeconds(messageDeadlineSeconds); + synchronized (outstandingAckHandlers) { + outstandingAckHandlers.add( + new ExtensionJob( + Instant.ofEpochMilli(clock.millisTime()), + expiration, + INITIAL_ACK_DEADLINE_EXTENSION_SECONDS, + ackHandlers)); + } + setupNextAckDeadlineExtensionAlarm(expiration); + + synchronized (outstandingMessageBatches) { + outstandingMessageBatches.add(outstandingBatch); + } + processOutstandingBatches(); + } + + void processOutstandingBatches() { + while (true) { + boolean batchDone = false; + Runnable batchCallback = null; + OutstandingMessagesBatch.OutstandingMessage outstandingMessage; + synchronized (outstandingMessageBatches) { + OutstandingMessagesBatch nextBatch = outstandingMessageBatches.peek(); + if (nextBatch == null) { + return; + } + outstandingMessage = nextBatch.messages.peek(); + if (outstandingMessage == null) { + return; + } + try { + // This is a non-blocking flow controller. + flowController.reserve( + 1, outstandingMessage.receivedMessage().getMessage().getSerializedSize()); + } catch (FlowController.MaxOutstandingElementCountReachedException + | FlowController.MaxOutstandingRequestBytesReachedException flowControlException) { + return; + } catch (FlowControlException unexpectedException) { + throw new IllegalStateException("Flow control unexpected exception", unexpectedException); + } + nextBatch.messages.poll(); // We got a hold to the message already. + batchDone = nextBatch.messages.isEmpty(); + if (batchDone) { + outstandingMessageBatches.poll(); + batchCallback = nextBatch.doneCallback; + } + } + + final PubsubMessage message = outstandingMessage.receivedMessage().getMessage(); + final AckHandler ackHandler = outstandingMessage.ackHandler(); + final SettableFuture response = SettableFuture.create(); + final AckReplyConsumer consumer = + new AckReplyConsumer() { + @Override + public void ack() { + response.set(AckReply.ACK); + } + + @Override + public void nack() { + response.set(AckReply.NACK); + } + }; + Futures.addCallback(response, ackHandler); + + try { + receiver.receiveMessage(message, consumer); + } catch (Exception e) { + response.setException(e); + } + + if (batchDone) { + batchCallback.run(); + } + } + } + + private class AckDeadlineAlarm implements Runnable { + @Override + public void run() { + alarmsLock.lock(); + try { + nextAckDeadlineExtensionAlarmTime = Instant.ofEpochMilli(Long.MAX_VALUE); + ackDeadlineExtensionAlarm = null; + if (pendingAcksAlarm != null) { + pendingAcksAlarm.cancel(false); + pendingAcksAlarm = null; + } + } finally { + alarmsLock.unlock(); + } + + Instant now = Instant.ofEpochMilli(clock.millisTime()); + // Rounded to the next second, so we only schedule future alarms at the second + // resolution. + Instant cutOverTime = + Instant.ofEpochMilli( + ((long) Math.ceil(now.plus(ackExpirationPadding).plusMillis(500).toEpochMilli() / 1000.0)) + * 1000L); + logger.log( + Level.FINER, + "Running alarm sent outstanding acks, at time: {0}, with cutover time: {1}, padding: {2}", + new Object[] {now, cutOverTime, ackExpirationPadding}); + Instant nextScheduleExpiration = null; + List modifyAckDeadlinesToSend = new ArrayList<>(); + + // Holding area for jobs we'll put back into the queue + // so we don't process the same job twice. + List renewJobs = new ArrayList<>(); + + synchronized (outstandingAckHandlers) { + while (!outstandingAckHandlers.isEmpty() + && outstandingAckHandlers.peek().expiration.compareTo(cutOverTime) <= 0) { + ExtensionJob job = outstandingAckHandlers.poll(); + + if (maxAckExtensionPeriod.toMillis() > 0 + && job.creation.plus(maxAckExtensionPeriod).compareTo(now) <= 0) { + // The job has expired, according to the maxAckExtensionPeriod, we are just going to + // drop it. + continue; + } + + // If a message has already been acked, remove it, nothing to do. + for (int i = 0; i < job.ackHandlers.size(); ) { + if (job.ackHandlers.get(i).acked.get()) { + Collections.swap(job.ackHandlers, i, job.ackHandlers.size() - 1); + job.ackHandlers.remove(job.ackHandlers.size() - 1); + } else { + i++; + } + } + + if (job.ackHandlers.isEmpty()) { + continue; + } + + job.extendExpiration(now); + long extensionMillis = Duration.between(now, job.expiration).toMillis(); + int extensionSeconds = Ints + .saturatedCast(TimeUnit.MILLISECONDS.toSeconds(extensionMillis)); + PendingModifyAckDeadline pendingModAckDeadline = + new PendingModifyAckDeadline(extensionSeconds); + for (AckHandler ackHandler : job.ackHandlers) { + pendingModAckDeadline.addAckId(ackHandler.ackId); + } + modifyAckDeadlinesToSend.add(pendingModAckDeadline); + renewJobs.add(job); + } + outstandingAckHandlers.addAll(renewJobs); + + if (!outstandingAckHandlers.isEmpty()) { + nextScheduleExpiration = outstandingAckHandlers.peek().expiration; + } + } + + extendAckDeadlines(modifyAckDeadlinesToSend); + + if (nextScheduleExpiration != null) { + logger.log( + Level.FINER, + "Scheduling based on outstanding, at time: {0}, next scheduled time: {1}", + new Object[] {now, nextScheduleExpiration}); + setupNextAckDeadlineExtensionAlarm(nextScheduleExpiration); + } + } + } + + private void setupNextAckDeadlineExtensionAlarm(Instant expiration) { + Instant possibleNextAlarmTime = expiration.minus(ackExpirationPadding); + alarmsLock.lock(); + try { + if (nextAckDeadlineExtensionAlarmTime.isAfter(possibleNextAlarmTime)) { + logger.log( + Level.FINER, + "Scheduling next alarm time: {0}, previous alarm time: {1}", + new Object[] {possibleNextAlarmTime, nextAckDeadlineExtensionAlarmTime}); + if (ackDeadlineExtensionAlarm != null) { + logger.log(Level.FINER, "Canceling previous alarm"); + ackDeadlineExtensionAlarm.cancel(false); + } + + nextAckDeadlineExtensionAlarmTime = possibleNextAlarmTime; + + ackDeadlineExtensionAlarm = + systemExecutor.schedule( + new AckDeadlineAlarm(), + nextAckDeadlineExtensionAlarmTime.toEpochMilli() - clock.millisTime(), + TimeUnit.MILLISECONDS); + } + + } finally { + alarmsLock.unlock(); + } + } + + void acknowledgePendingMessages(boolean sync) { + Map acksToSend = new HashMap<>(); + synchronized (pendingAcks) { + if (!pendingAcks.isEmpty()) { + for (Entry pair : pendingAcks.entrySet()){ + acksToSend.put(pair.getKey(), pair.getValue()); + } + logger.log(Level.FINER, "Sending {0} acks", acksToSend.size()); + } + } + + if(sync) { + commitSync(acksToSend); + } else { + commitAsync(acksToSend); + } + } + + private void commitAsync(Map acksToSend) { + ListenableFuture> listListenableFuture = ackProcessor + .sendAckOperations(new ArrayList<>(acksToSend.keySet()), Collections.emptyList()); + + Futures.addCallback(listListenableFuture, new FutureCallback>() { + @Override + public void onSuccess(@Nullable List empties) { + handleSuccessfulAck(acksToSend); + } + + @Override + public void onFailure(Throwable throwable) { + logger.log(Level.WARNING, "Failed to commit async", throwable); + } + }); + } + + private void commitSync(Map acksToSend) { + ListenableFuture> listListenableFuture = ackProcessor + .sendAckOperations(new ArrayList<>(acksToSend.keySet()), Collections.emptyList()); + + try { + listListenableFuture.get(); + handleSuccessfulAck(acksToSend); + } catch (InterruptedException | ExecutionException e) { + if(StatusUtil.isRetryable(e)) { + sleep(this.retryBackoffMs); + commitSync(acksToSend); + } else { + throw new KafkaException(e); + } + } + } + + private void handleSuccessfulAck( + Map acksToSend) { + synchronized (pendingAcks) { + for(Entry entry : acksToSend.entrySet()) { + entry.getValue().acked.getAndSet(true); + pendingAcks.remove(entry.getKey()); + } + } + } + + private void sleep(long ms) { + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + + private void extendAckDeadlines(List ackDeadlineExtensions) { + List modifyAckDeadlinesToSend = + Lists.newArrayList(ackDeadlineExtensions); + PendingModifyAckDeadline nacksToSend = new PendingModifyAckDeadline(0); + synchronized (pendingNacks) { + if (!pendingNacks.isEmpty()) { + try { + for (String ackId : pendingNacks) { + nacksToSend.addAckId(ackId); + } + logger.log(Level.FINER, "Sending {0} nacks", pendingNacks.size()); + } finally { + pendingNacks.clear(); + } + modifyAckDeadlinesToSend.add(nacksToSend); + } + } + ackProcessor.sendAckOperations(Collections.emptyList(), modifyAckDeadlinesToSend); + + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageReceiver.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageReceiver.java new file mode 100644 index 00000000..75d4b154 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageReceiver.java @@ -0,0 +1,48 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.pubsub.v1.PubsubMessage; + +/** This interface can be implemented by users of {@link Subscriber} to receive messages. */ +public interface MessageReceiver { + /** + * Called when a message is received by the subscriber. The implementation must arrange for {@link + * AckReplyConsumer#ack()} or {@link + * AckReplyConsumer#nack()} to be called after processing the {@code message}. + * + *

This {@code MessageReceiver} passes all messages to a {@code BlockingQueue}. + * This method can be called concurrently from multiple threads, + * so it is important that the queue be thread-safe. + * + * This example is for illustration. Implementations may directly process messages + * instead of sending them to queues. + * 

 {@code
+   * MessageReceiver receiver = new MessageReceiver() {
+   *   public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
+   *     if (blockingQueue.offer(message)) {
+   *       consumer.ack();
+   *     } else {
+   *       consumer.nack();
+   *     }
+   *   }
+   * };
+   * }
+ * + */ + void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer); +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageWaiter.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageWaiter.java new file mode 100644 index 00000000..5915407e --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageWaiter.java @@ -0,0 +1,67 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.common.annotations.VisibleForTesting; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A barrier kind of object that helps to keep track and synchronously wait on pending messages. + */ +class MessageWaiter { + private int pendingMessages; + + MessageWaiter() { + pendingMessages = 0; + } + + public synchronized void incrementPendingMessages(int messages) { + this.pendingMessages += messages; + if (pendingMessages == 0) { + notifyAll(); + } + } + + public synchronized void waitNoMessages() { + waitNoMessages(new AtomicBoolean()); + } + + @VisibleForTesting + synchronized void waitNoMessages(AtomicBoolean waitReached) { + boolean interrupted = false; + try { + while (pendingMessages > 0) { + try { + waitReached.set(true); + wait(); + } catch (InterruptedException e) { + // Ignored, uninterruptibly. + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + @VisibleForTesting + public int pendingMessages() { + return pendingMessages; + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java new file mode 100644 index 00000000..b9ee10c1 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java @@ -0,0 +1,165 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.api.core.AbstractApiService; +import com.google.api.core.ApiClock; +import com.google.api.gax.batching.FlowController; +import com.google.api.gax.core.Distribution; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.Empty; +import com.google.pubsub.clients.consumer.ack.MessageDispatcher.PendingModifyAckDeadline; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.ModifyAckDeadlineRequest; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; +import com.google.pubsub.v1.Subscription; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.common.KafkaException; +import org.threeten.bp.Duration; + +/** + * Implementation of {@link MessageDispatcher.AckProcessor} based on Cloud Pub/Sub pull and acknowledge operations. + */ +final class PollingSubscriberConnection extends AbstractApiService implements MessageDispatcher.AckProcessor { + + private final Subscription subscription; + private final SubscriberFutureStub stub; + private final MessageDispatcher messageDispatcher; + private final int maxDesiredPulledMessages; + private final Integer maxPerRequestChanges; + private final Integer extensionAckTimeoutMs; + + PollingSubscriberConnection( + Subscription subscription, + MessageReceiver receiver, + Duration ackExpirationPadding, + Duration maxAckExtensionPeriod, + Distribution ackLatencyDistribution, + SubscriberFutureStub stub, + FlowController flowController, + Long maxDesiredPulledMessages, + ScheduledExecutorService systemExecutor, + ApiClock clock, + Long retryBackoffMs, + Integer maxPerRequestChanges, + Integer extensionAckTimeoutMs) { + this.subscription = subscription; + this.stub = stub; + messageDispatcher = + new MessageDispatcher( + receiver, + this, + ackExpirationPadding, + maxAckExtensionPeriod, + ackLatencyDistribution, + flowController, + systemExecutor, + clock, + retryBackoffMs); + + this.maxPerRequestChanges = maxPerRequestChanges; + + messageDispatcher.setMessageDeadlineSeconds(subscription.getAckDeadlineSeconds()); + this.maxDesiredPulledMessages = Ints.saturatedCast(maxDesiredPulledMessages); + this.extensionAckTimeoutMs = extensionAckTimeoutMs; + } + + PullResponse pullMessages(final long timeout) throws ExecutionException, InterruptedException { + if (!isAlive()) { + throw new KafkaException("Subscriber is not alive"); + } + + ListenableFuture pullResult = + stub.withDeadlineAfter(timeout, TimeUnit.MILLISECONDS).pull( + PullRequest.newBuilder() + .setSubscription(subscription.getName()) + .setMaxMessages(maxDesiredPulledMessages) + .setReturnImmediately(true) + .build()); + + PullResponse response = pullResult.get(); + + messageDispatcher.processReceivedMessages(response.getReceivedMessagesList(), () -> {}); + + //on non-retry-fail - stop dispatcher? + return response; + } + + private boolean isAlive() { + // Read state only once. Because of threading, different calls can give different results. + State state = state(); + return state == State.RUNNING || state == State.STARTING; + } + + void commit(boolean sync, OffsetCommitCallback callback) { + messageDispatcher.acknowledgePendingMessages(sync); + } + + @Override + public ListenableFuture> sendAckOperations( + List acksToSend, List ackDeadlineExtensions) { + // Send the modify ack deadlines in batches as not to exceed the max request + // size. + for (MessageDispatcher.PendingModifyAckDeadline modifyAckDeadline : ackDeadlineExtensions) { + for (List ackIdChunk : + Lists.partition(modifyAckDeadline.ackIds, maxPerRequestChanges)) { + stub.withDeadlineAfter(extensionAckTimeoutMs, TimeUnit.MILLISECONDS) + .modifyAckDeadline( + ModifyAckDeadlineRequest.newBuilder() + .setSubscription(subscription.getName()) + .addAllAckIds(ackIdChunk) + .setAckDeadlineSeconds(modifyAckDeadline.deadlineExtensionSeconds) + .build()); + } + } + + List> acknowledges = new ArrayList<>(); + + for (List ackChunk : Lists.partition(acksToSend, maxPerRequestChanges)) { + ListenableFuture acknowledge = stub.withDeadlineAfter(extensionAckTimeoutMs, TimeUnit.MILLISECONDS) + .acknowledge( + AcknowledgeRequest.newBuilder() + .setSubscription(subscription.getName()) + .addAllAckIds(ackChunk) + .build()); + acknowledges.add(acknowledge); + } + + return Futures.allAsList(acknowledges); + } + + @Override + protected void doStart() { + notifyStarted(); + } + + @Override + protected void doStop() { + messageDispatcher.stop(); + notifyStopped(); + } +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/StatusUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/StatusUtil.java new file mode 100644 index 00000000..f66bdb93 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/StatusUtil.java @@ -0,0 +1,49 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +/** Utilities for handling gRPC {@link Status}. */ +final class StatusUtil { + private StatusUtil() { + // Static class, not instantiable. + } + + static boolean isRetryable(Throwable error) { + if (!(error instanceof StatusRuntimeException)) { + return true; + } + StatusRuntimeException statusRuntimeException = (StatusRuntimeException) error; + switch (statusRuntimeException.getStatus().getCode()) { + case DEADLINE_EXCEEDED: + case INTERNAL: + case CANCELLED: + case RESOURCE_EXHAUSTED: + return true; + case UNAVAILABLE: + if (statusRuntimeException.getMessage().contains("Server shutdownNow invoked")) { + return false; + } else { + return true; + } + default: + return false; + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java new file mode 100644 index 00000000..5eaf17ad --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java @@ -0,0 +1,374 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.api.core.AbstractApiService; +import com.google.api.core.ApiClock; +import com.google.api.core.ApiService; +import com.google.api.core.CurrentMillisClock; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import com.google.api.gax.core.Distribution; +import com.google.api.gax.grpc.ExecutorProvider; +import com.google.api.gax.grpc.FixedExecutorProvider; +import com.google.api.gax.grpc.InstantiatingExecutorProvider; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import org.threeten.bp.Duration; + +/** + * A Cloud Pub/Sub subscriber that is + * associated with a specific subscription at creation. + * + *

A {@link Subscriber} allows you to provide an implementation of a {@link MessageReceiver + * receiver} to which messages are going to be delivered as soon as they are received by the + * subscriber. The delivered messages then can be {@link AckReplyConsumer#ack() acked} or {@link + * AckReplyConsumer#nack() nacked} at will as they get processed by the receiver. Nacking a messages + * implies a later redelivery of such message. + * + *

The subscriber handles the ack management, by automatically extending the ack deadline while + * the message is being processed, to then issue the ack or nack of such message when the processing + * is done. Note: message redelivery is still possible. + * + *

It also provides customizable options that control: + * + *

    + *
  • Ack deadline extension: such as the amount of time ahead to trigger the extension of + * message acknowledgement expiration. + *
  • Flow control: such as the maximum outstanding messages or maximum outstanding bytes to keep + * in memory before the receiver either ack or nack them. + *
+ * + *

{@link Subscriber} will use the credentials set on the channel, which uses application default + * credentials through {@link GoogleCredentials#getApplicationDefault} by default. + */ +public class Subscriber extends AbstractApiService { + private static final int MAX_ACK_DEADLINE_SECONDS = 600; + + private static final ScheduledExecutorService SHARED_SYSTEM_EXECUTOR = + InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(6).build().getExecutor(); + + private static final Logger logger = Logger.getLogger(Subscriber.class.getName()); + + private final Subscription subscription; + private final FlowControlSettings flowControlSettings; + private final Duration ackExpirationPadding; + @Nullable private final ScheduledExecutorService alarmsExecutor; + private final PollingSubscriberConnection pollingSubscriberConnection; + private final ApiClock clock; + private final List closeables = new ArrayList<>(); + private long nextCommitTime; + private Boolean autoCommit; + private final Integer autoCommitIntervalMs; + + private Subscriber(Builder builder) { + MessageReceiver receiver = builder.receiver; + Duration maxAckExtensionPeriod = builder.maxAckExtensionPeriod; + Long maxPullRecords = builder.maxPullRecords; + Long retryBackoffMs = builder.retryBackoffMs; + Integer maxPerRequestChanges = builder.maxPerRequestChanges; + Integer ackRequestTimeoutMs = builder.ackRequestTimeoutMs; + + flowControlSettings = builder.flowControlSettings; + subscription = builder.subscription; + ackExpirationPadding = builder.ackExpirationPadding; + clock = builder.clock.isPresent() ? builder.clock.get() : CurrentMillisClock.getDefaultClock(); + autoCommitIntervalMs = builder.autoCommitIntervalMs; + autoCommit = builder.autoCommit; + + if(this.autoCommit) { + this.nextCommitTime = clock.millisTime() + autoCommitIntervalMs; + } + + FlowController flowController = new FlowController( + builder + .flowControlSettings + .toBuilder() + .setLimitExceededBehavior(LimitExceededBehavior.ThrowException) + .build()); + + alarmsExecutor = builder.systemExecutorProvider.getExecutor(); + if (builder.systemExecutorProvider.shouldAutoClose()) { + closeables.add( + new AutoCloseable() { + @Override + public void close() throws IOException { + alarmsExecutor.shutdown(); + } + }); + } + + SubscriberFutureStub stub = builder.subscriberFutureStub; + + Distribution ackLatencyDistribution = new Distribution(MAX_ACK_DEADLINE_SECONDS + 1); + this.pollingSubscriberConnection = new PollingSubscriberConnection( + subscription, + receiver, + ackExpirationPadding, + maxAckExtensionPeriod, + ackLatencyDistribution, + stub, + flowController, + maxPullRecords, + alarmsExecutor, + clock, + retryBackoffMs, + maxPerRequestChanges, + ackRequestTimeoutMs); + } + + /** + * Constructs a new {@link Builder}. + * + *

Once {@link Builder#build} is called a gRPC stub will be created for use of the {@link + * Subscriber}. + * + * @param subscription Cloud Pub/Sub subscription to bind the subscriber to + * @param receiver an implementation of {@link MessageReceiver} used to process the received + * messages + */ + public static Builder defaultBuilder(Subscription subscription, MessageReceiver receiver) { + return new Builder(subscription, receiver); + } + + public Subscription getSubscription() { + return this.subscription; + } + + /** Subscription which the subscriber is subscribed to. */ + public SubscriptionName getSubscriptionName() { + return subscription.getNameAsSubscriptionName(); + } + + /** Acknowledgement expiration padding. See {@link Builder#setAckExpirationPadding}. */ + public Duration getAckExpirationPadding() { + return ackExpirationPadding; + } + + /** The flow control settings the Subscriber is configured with. */ + public FlowControlSettings getFlowControlSettings() { + return flowControlSettings; + } + + /** + * Initiates service startup and returns immediately. + * + *

Example of receiving a specific number of messages. + * + * 

{@code
+   * Subscriber subscriber = Subscriber.defaultBuilder(subscription, receiver).build();
+   * subscriber.addListener(new Subscriber.Listener() {
+   *   public void failed(Subscriber.State from, Throwable failure) {
+   *     // Handle error.
+   *   }
+   * }, executor);
+   * subscriber.startAsync();
+   *
+   * // Wait for a stop signal.
+   * done.get();
+   * subscriber.stopAsync().awaitTerminated();
+   * }
+ */ + @Override + public ApiService startAsync() { + // Override only for the docs. + return super.startAsync(); + } + + public void commit(boolean sync) { + pollingSubscriberConnection.commit(sync, null); + } + + + @Override + public void doStart() { + logger.log(Level.FINE, "Starting subscriber group."); + pollingSubscriberConnection.startAsync().awaitRunning(); + notifyStarted(); + } + + @Override + public void doStop() { + // stop connection is no-op if connections haven't been started. + stopAllPollingConnection(); + try { + for (AutoCloseable closeable : closeables) { + closeable.close(); + } + notifyStopped(); + } catch (Exception e) { + notifyFailed(e); + } + } + + public PullResponse pull(final long timeout) throws IOException, ExecutionException, InterruptedException { + long now = clock.millisTime(); + synchronized (pollingSubscriberConnection) { + if(this.autoCommit && this.nextCommitTime <= now) { + pollingSubscriberConnection.commit(false, null); + this.nextCommitTime = now + this.autoCommitIntervalMs; + } + return pollingSubscriberConnection.pullMessages(timeout); + } + } + + private void stopAllPollingConnection() { + pollingSubscriberConnection.stopAsync().awaitTerminated(); + } + + /** Builder of {@link Subscriber Subscribers}. */ + public static final class Builder { + private static final Duration MIN_ACK_EXPIRATION_PADDING = Duration.ofMillis(100); + private static final Duration DEFAULT_ACK_EXPIRATION_PADDING = Duration.ofMillis(500); + private static final long DEFAULT_MEMORY_PERCENTAGE = 20; + + MessageReceiver receiver; + + Duration ackExpirationPadding = DEFAULT_ACK_EXPIRATION_PADDING; + Duration maxAckExtensionPeriod; + + FlowControlSettings flowControlSettings = + FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes( + Runtime.getRuntime().maxMemory() * DEFAULT_MEMORY_PERCENTAGE / 100L) + .build(); + + ExecutorProvider systemExecutorProvider = FixedExecutorProvider.create(SHARED_SYSTEM_EXECUTOR); + Optional clock = Optional.absent(); + private final Subscription subscription; + private Long maxPullRecords; + private Boolean autoCommit; + private Integer autoCommitIntervalMs; + private SubscriberFutureStub subscriberFutureStub; + private Long retryBackoffMs; + private Integer maxPerRequestChanges; + private Integer ackRequestTimeoutMs; + + Builder(Subscription subscription, MessageReceiver receiver) { + this.subscription = subscription; + this.receiver = receiver; + } + + /** Sets the flow control settings. */ + public Builder setFlowControlSettings(FlowControlSettings flowControlSettings) { + this.flowControlSettings = Preconditions.checkNotNull(flowControlSettings); + return this; + } + + /** + * Set acknowledgement expiration padding. + * + *

This is the time accounted before a message expiration is to happen, so the {@link + * Subscriber} is able to send an ack extension beforehand. + * + *

This padding duration is configurable so you can account for network latency. A reasonable + * number must be provided so messages don't expire because of network latency between when the + * ack extension is required and when it reaches the Pub/Sub service. + * + * @param ackExpirationPadding must be greater or equal to {@link #MIN_ACK_EXPIRATION_PADDING} + */ + public Builder setAckExpirationPadding(Duration ackExpirationPadding) { + Preconditions.checkArgument(ackExpirationPadding.compareTo(MIN_ACK_EXPIRATION_PADDING) >= 0); + this.ackExpirationPadding = ackExpirationPadding; + return this; + } + + /** + * Set the maximum period a message ack deadline will be extended. + * + *

It is recommended to set this value to a reasonable upper bound of the subscriber time to + * process any message. This maximum period avoids messages to be locked by a subscriber + * in cases when the ack reply is lost. + * + *

A zero duration effectively disables auto deadline extensions. + */ + public Builder setMaxAckExtensionPeriod(Integer maxAckExtensionPeriod) { + Preconditions.checkArgument(maxAckExtensionPeriod >= 0); + this.maxAckExtensionPeriod = Duration.ofSeconds(maxAckExtensionPeriod); + return this; + } + + /** + * Gives the ability to set a custom executor for polling and managing lease extensions. If none + * is provided a shared one will be used by all {@link Subscriber} instances. + */ + public Builder setSystemExecutorProvider(ExecutorProvider executorProvider) { + this.systemExecutorProvider = Preconditions.checkNotNull(executorProvider); + return this; + } + + public Builder setAutoCommit(Boolean autoCommit) { + this.autoCommit = autoCommit; + return this; + } + + /** Gives the ability to set a custom clock. */ + public Builder setClock(ApiClock clock) { + this.clock = Optional.of(clock); + return this; + } + + public Builder setMaxPullRecords(Long maxPullRecords) { + this.maxPullRecords = maxPullRecords; + return this; + } + + public Builder setAutoCommitIntervalMs(Integer autoCommitIterval) { + this.autoCommitIntervalMs = autoCommitIterval; + return this; + } + + public Builder setSubscriberFutureStub(SubscriberFutureStub subscriberFutureStub) { + this.subscriberFutureStub = subscriberFutureStub; + return this; + } + + public Builder setRetryBackoffMs(Long retryBackoffMs) { + this.retryBackoffMs = retryBackoffMs; + return this; + } + + public Builder setMaxPerRequestChanges(Integer maxPerRequestChanges) { + this.maxPerRequestChanges = maxPerRequestChanges; + return this; + } + + public Builder setAckRequestTimeoutMs(Integer ackRequestTimeoutMs) { + this.ackRequestTimeoutMs = ackRequestTimeoutMs; + return this; + } + + public Subscriber build() { + return new Subscriber(this); + } + } +} + diff --git a/pubsub-mapped-api/src/main/resources/log4j.properties b/pubsub-mapped-api/src/main/resources/log4j.properties index ff0cb5bd..1c828aa6 100644 --- a/pubsub-mapped-api/src/main/resources/log4j.properties +++ b/pubsub-mapped-api/src/main/resources/log4j.properties @@ -1,4 +1,4 @@ -log4j.rootLogger=DEBUG, CONSOLE +log4j.rootLogger=ERROR, CONSOLE log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java index 89cfe6f4..d0eb3ed9 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -78,6 +78,7 @@ public void subscribeOnlyGet() { Set subscribed = consumer.subscription(); assertEquals(topics, subscribed); + consumer.close(); } @Test @@ -90,6 +91,7 @@ public void subscribeNoExistingSubscriptionsAllowCreation() { Set subscribed = consumer.subscription(); assertEquals(topics, subscribed); + consumer.close(); } @Test @@ -103,6 +105,7 @@ public void subscribeNoExistingSubscriptionsDontAllowCreation() { } catch (KafkaException e) { //expected } + consumer.close(); } @Test @@ -114,6 +117,7 @@ public void unsubscribe() { consumer.unsubscribe(); assertTrue(consumer.subscription().isEmpty()); + consumer.close(); } @Test @@ -128,6 +132,7 @@ public void resubscribe() { Set subscribed = consumer.subscription(); assertEquals(topics, subscribed); + consumer.close(); } @Test @@ -141,6 +146,7 @@ public void subscribeWithRecurringTopics() { Set subscribed = consumer.subscription(); assertEquals(topics, subscribed); + consumer.close(); } @Test @@ -155,6 +161,7 @@ public void patternSubscription() { Set subscribed = consumer.subscription(); Set expectedTopics = new HashSet<>(Arrays.asList("thisis123cat", "funnycats000cat")); assertEquals(expectedTopics, subscribed); + consumer.close(); } @Test @@ -167,6 +174,7 @@ public void emptyPatternFails() { } catch (IllegalArgumentException e) { assertEquals("Topic pattern to subscribe to cannot be null", e.getMessage()); } + consumer.close(); } @Test @@ -179,6 +187,7 @@ public void nullTopicListFails() { } catch (IllegalArgumentException e) { assertEquals("Topic collection to subscribe to cannot be null", e.getMessage()); } + consumer.close(); } @Test @@ -191,6 +200,7 @@ public void emptyTopicFails() { } catch (IllegalArgumentException e) { assertEquals("Topic collection to subscribe to cannot contain null or empty topic", e.getMessage()); } + consumer.close(); } @Test @@ -202,6 +212,7 @@ public void noSubscriptionsPollFails() { } catch (IllegalStateException e) { assertEquals("Consumer is not subscribed to any topics", e.getMessage()); } + consumer.close(); } @Test @@ -213,6 +224,7 @@ public void negativePollTimeoutFails() { } catch (IllegalArgumentException e) { assertEquals("Timeout must not be negative", e.getMessage()); } + consumer.close(); } @Test @@ -235,6 +247,7 @@ public void deserializeProperly() throws ExecutionException, InterruptedExceptio assertEquals(consumerRecord.count(), 1); assertEquals(value, next.value()); assertEquals(key, next.key()); + consumer.close(); } @Test @@ -274,6 +287,10 @@ private Properties getTestProperties(boolean allowesCreation) { return properties; } + private String getTopicPrefixString(String topicName) { + return "projects/" + System.getenv("GOOGLE_CLOUD_PROJECT") + "/topics/" + topicName; + } + static class SubscriberGetImpl extends SubscriberImplBase { @Override @@ -283,7 +300,7 @@ public void createSubscription(Subscription request, StreamObserver responseObserver) { - Subscription s = Subscription.newBuilder().setName("name").setTopic("projects/null/topics/topic").build(); + Subscription s = Subscription.newBuilder().setName("projects/null/subscriptions/name").setTopic("projects/null/topics/topic").build(); responseObserver.onNext(s); responseObserver.onCompleted(); } @@ -332,7 +349,7 @@ public void acknowledge(AcknowledgeRequest request, StreamObserver respon static class SubscriberNoExistingSubscriptionsImpl extends SubscriberImplBase { @Override public void createSubscription(Subscription request, StreamObserver responseObserver) { - Subscription s = Subscription.newBuilder().setName("name").setTopic("projects/null/topics/topic").build(); + Subscription s = Subscription.newBuilder().setName("projects/null/subscriptions/name").setTopic("projects/null/topics/topic").build(); responseObserver.onNext(s); responseObserver.onCompleted(); } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeClock.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeClock.java new file mode 100644 index 00000000..76525c94 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeClock.java @@ -0,0 +1,42 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.api.core.ApiClock; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** A Clock to help with testing time-based logic. */ +public class FakeClock implements ApiClock { + + private final AtomicLong millis = new AtomicLong(); + + // Advances the clock value by {@code time} in {@code timeUnit}. + public void advance(long time, TimeUnit timeUnit) { + millis.addAndGet(timeUnit.toMillis(time)); + } + + @Override + public long nanoTime() { + return millisTime() * 1000_000L; + } + + @Override + public long millisTime() { + return millis.get(); + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeScheduledExecutorService.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeScheduledExecutorService.java new file mode 100644 index 00000000..dedd5485 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeScheduledExecutorService.java @@ -0,0 +1,353 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + + +import com.google.api.core.ApiClock; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.SettableFuture; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.PriorityQueue; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.Callable; +import java.util.concurrent.Delayed; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.threeten.bp.Duration; +import org.threeten.bp.Instant; + +/** + * Fake implementation of {@link ScheduledExecutorService} that allows tests control the reference + * time of the executor and decide when to execute any outstanding task. + */ +public class FakeScheduledExecutorService extends AbstractExecutorService + implements ScheduledExecutorService { + + private final AtomicBoolean shutdown = new AtomicBoolean(false); + private final PriorityQueue> pendingCallables = new PriorityQueue<>(); + private final FakeClock clock = new FakeClock(); + private final Deque expectedWorkQueue = new LinkedList<>(); + + public ApiClock getClock() { + return clock; + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + return schedulePendingCallable( + new PendingCallable<>( + Duration.ofMillis(unit.toMillis(delay)), command, PendingCallableType.NORMAL)); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + return schedulePendingCallable( + new PendingCallable<>( + Duration.ofMillis(unit.toMillis(delay)), callable, PendingCallableType.NORMAL)); + } + + @Override + public ScheduledFuture scheduleAtFixedRate( + Runnable command, long initialDelay, long period, TimeUnit unit) { + return schedulePendingCallable( + new PendingCallable<>( + Duration.ofMillis(unit.toMillis(initialDelay)), command, PendingCallableType.FIXED_RATE)); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay( + Runnable command, long initialDelay, long delay, TimeUnit unit) { + return schedulePendingCallable( + new PendingCallable<>( + Duration.ofMillis(unit.toMillis(initialDelay)), command, PendingCallableType.FIXED_DELAY)); + } + + /** + * This allows for adding expectations on future work to be scheduled ( + * {@link FakeScheduledExecutorService#schedule} + * or {@link FakeScheduledExecutorService#scheduleAtFixedRate} + * or {@link FakeScheduledExecutorService#scheduleWithFixedDelay}) based on its delay. + */ + public void setupScheduleExpectation(Duration delay) { + synchronized (expectedWorkQueue) { + expectedWorkQueue.add(delay); + } + } + + /** + * Blocks the current thread until all the work + * {@link FakeScheduledExecutorService#setupScheduleExpectation(Duration) expected} has been + * scheduled in the executor. + */ + public void waitForExpectedWork() { + synchronized (expectedWorkQueue) { + while (!expectedWorkQueue.isEmpty()) { + try { + expectedWorkQueue.wait(); + } catch (InterruptedException e) { + // Wait uninterruptibly + } + } + } + } + + /** + * This will advance the reference time of the executor and execute (in the same thread) any + * outstanding callable which execution time has passed. + */ + public void advanceTime(Duration toAdvance) { + clock.advance(toAdvance.toMillis(), TimeUnit.MILLISECONDS); + work(); + } + + private void work() { + Instant cmpTime = Instant.ofEpochMilli(clock.millisTime()); + + for (;;) { + PendingCallable callable = null; + synchronized (pendingCallables) { + if (pendingCallables.isEmpty() + || pendingCallables.peek().getScheduledTime().isAfter(cmpTime)) { + break; + } + callable = pendingCallables.poll(); + } + if (callable != null) { + try { + callable.call(); + } catch (Exception e) { + // We ignore any callable exception, which should be set to the future but not relevant to + // advanceTime. + } + } + } + + synchronized (pendingCallables) { + if (shutdown.get() && pendingCallables.isEmpty()) { + pendingCallables.notifyAll(); + } + } + } + + @Override + public void shutdown() { + if (shutdown.getAndSet(true)) { + throw new IllegalStateException("This executor has been shutdown already"); + } + } + + @Override + public List shutdownNow() { + if (shutdown.getAndSet(true)) { + throw new IllegalStateException("This executor has been shutdown already"); + } + List pending = new ArrayList<>(); + for (final PendingCallable pendingCallable : pendingCallables) { + pending.add( + new Runnable() { + @Override + public void run() { + pendingCallable.call(); + } + }); + } + synchronized (pendingCallables) { + pendingCallables.notifyAll(); + pendingCallables.clear(); + } + return pending; + } + + @Override + public boolean isShutdown() { + return shutdown.get(); + } + + @Override + public boolean isTerminated() { + return pendingCallables.isEmpty(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + synchronized (pendingCallables) { + if (pendingCallables.isEmpty()) { + return true; + } + pendingCallables.wait(unit.toMillis(timeout)); + return pendingCallables.isEmpty(); + } + } + + @Override + public void execute(Runnable command) { + if (shutdown.get()) { + throw new IllegalStateException("This executor has been shutdown"); + } + command.run(); + } + + ScheduledFuture schedulePendingCallable(PendingCallable callable) { + if (shutdown.get()) { + throw new IllegalStateException("This executor has been shutdown"); + } + synchronized (pendingCallables) { + pendingCallables.add(callable); + } + work(); + synchronized (expectedWorkQueue) { + // We compare by the callable delay in order decide when to remove expectations from the + // expected work queue, i.e. only the expected work that matches the delay of the scheduled + // callable is removed from the queue. + if (!expectedWorkQueue.isEmpty() && expectedWorkQueue.peek().equals(callable.delay)) { + expectedWorkQueue.poll(); + } + expectedWorkQueue.notifyAll(); + } + + return callable.getScheduledFuture(); + } + + static enum PendingCallableType { + NORMAL, + FIXED_RATE, + FIXED_DELAY + } + + /** Class that saves the state of an scheduled pending callable. */ + class PendingCallable implements Comparable> { + Instant creationTime = Instant.ofEpochMilli(clock.millisTime()); + Duration delay; + Callable pendingCallable; + SettableFuture future = SettableFuture.create(); + AtomicBoolean cancelled = new AtomicBoolean(false); + AtomicBoolean done = new AtomicBoolean(false); + PendingCallableType type; + + PendingCallable(Duration delay, final Runnable runnable, PendingCallableType type) { + pendingCallable = + new Callable() { + @Override + public T call() throws Exception { + runnable.run(); + return null; + } + }; + this.type = type; + this.delay = delay; + } + + PendingCallable(Duration delay, Callable callable, PendingCallableType type) { + pendingCallable = callable; + this.type = type; + this.delay = delay; + } + + private Instant getScheduledTime() { + return creationTime.plus(delay); + } + + ScheduledFuture getScheduledFuture() { + return new ScheduledFuture() { + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(getScheduledTime().toEpochMilli() - clock.millisTime(), + TimeUnit.MILLISECONDS); + } + + @Override + public int compareTo(Delayed o) { + return Ints.saturatedCast( + getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS)); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + synchronized (this) { + cancelled.set(true); + return !done.get(); + } + } + + @Override + public boolean isCancelled() { + return cancelled.get(); + } + + @Override + public boolean isDone() { + return done.get(); + } + + @Override + public T get() throws InterruptedException, ExecutionException { + return future.get(); + } + + @Override + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return future.get(timeout, unit); + } + }; + } + + T call() { + T result = null; + synchronized (this) { + if (cancelled.get()) { + return null; + } + try { + result = pendingCallable.call(); + future.set(result); + } catch (Exception e) { + future.setException(e); + } finally { + switch (type) { + case NORMAL: + done.set(true); + break; + case FIXED_DELAY: + this.creationTime = Instant.ofEpochMilli(clock.millisTime()); + schedulePendingCallable(this); + break; + case FIXED_RATE: + this.creationTime = this.creationTime.plus(delay); + schedulePendingCallable(this); + break; + default: + // Nothing to do + } + } + } + return result; + } + + @Override + public int compareTo(PendingCallable other) { + return getScheduledTime().compareTo(other.getScheduledTime()); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeSubscriberServiceImpl.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeSubscriberServiceImpl.java new file mode 100644 index 00000000..089f0836 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/FakeSubscriberServiceImpl.java @@ -0,0 +1,180 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import com.google.api.client.util.Preconditions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Empty; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.GetSubscriptionRequest; +import com.google.pubsub.v1.ModifyAckDeadlineRequest; +import com.google.pubsub.v1.PublisherGrpc.PublisherImplBase; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberImplBase; +import com.google.pubsub.v1.Subscription; +import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A fake implementation of {@link PublisherImplBase}, that can be used to test clients of a Cloud + * Pub/Sub Publisher. + */ +class FakeSubscriberServiceImpl extends SubscriberImplBase { + private final AtomicBoolean subscriptionInitialized = new AtomicBoolean(false); + private String subscription = ""; + private final AtomicInteger messageAckDeadline = + new AtomicInteger(10); + private final AtomicInteger getSubscriptionCalled = new AtomicInteger(); + private final List acks = new ArrayList<>(); + private final List modAckDeadlines = new ArrayList<>(); + private final BlockingQueue pullResponses = new LinkedBlockingDeque<>(); + + public static final class ModifyAckDeadline { + private final String ackId; + private final long seconds; + + ModifyAckDeadline(String ackId, long seconds) { + Preconditions.checkNotNull(ackId); + this.ackId = ackId; + this.seconds = seconds; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ModifyAckDeadline)) { + return false; + } + ModifyAckDeadline other = (ModifyAckDeadline) obj; + return other.ackId.equals(this.ackId) && other.seconds == this.seconds; + } + + @Override + public int hashCode() { + return ackId.hashCode(); + } + + @Override + public String toString() { + return "Ack ID: " + ackId + ", deadline seconds: " + seconds; + } + } + + void enqueuePullResponse(PullResponse response) { + pullResponses.add(response); + } + + @Override + public void getSubscription( + GetSubscriptionRequest request, StreamObserver responseObserver) { + getSubscriptionCalled.incrementAndGet(); + responseObserver.onNext( + Subscription.newBuilder() + .setName(request.getSubscription()) + .setAckDeadlineSeconds(messageAckDeadline.get()) + .setTopic("fake-topic") + .build()); + responseObserver.onCompleted(); + } + + /** Returns the number of times getSubscription is called. */ + @VisibleForTesting + int getSubscriptionCalledCount() { + return getSubscriptionCalled.get(); + } + + @Override + public void pull(PullRequest request, StreamObserver responseObserver) { + try { + responseObserver.onNext(pullResponses.take()); + responseObserver.onCompleted(); + } catch (InterruptedException e) { + responseObserver.onError(e); + } + } + + @Override + public void acknowledge( + AcknowledgeRequest request, io.grpc.stub.StreamObserver responseObserver) { + addReceivedAcks(request.getAckIdsList()); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + + @Override + public void modifyAckDeadline( + ModifyAckDeadlineRequest request, StreamObserver responseObserver) { + for (String ackId : request.getAckIdsList()) { + addReceivedModifyAckDeadline(new ModifyAckDeadline(ackId, request.getAckDeadlineSeconds())); + } + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + + List waitAndConsumeReceivedAcks(int expectedCount) throws InterruptedException { + synchronized (acks) { + while (acks.size() < expectedCount) { + acks.wait(); + } + List receivedAcksCopy = ImmutableList.copyOf(acks.subList(0, expectedCount)); + acks.removeAll(receivedAcksCopy); + return receivedAcksCopy; + } + } + + List waitAndConsumeModifyAckDeadlines(int expectedCount) + throws InterruptedException { + synchronized (modAckDeadlines) { + while (modAckDeadlines.size() < expectedCount) { + modAckDeadlines.wait(); + } + List modAckDeadlinesCopy = + ImmutableList.copyOf(modAckDeadlines.subList(0, expectedCount)); + modAckDeadlines.removeAll(modAckDeadlinesCopy); + return modAckDeadlinesCopy; + } + } + + List getAcks() { + return acks; + } + + List getModifyAckDeadlines() { + return modAckDeadlines; + } + + private void addReceivedAcks(Collection newAckIds) { + synchronized (acks) { + acks.addAll(newAckIds); + acks.notifyAll(); + } + } + + private void addReceivedModifyAckDeadline(ModifyAckDeadline newAckDeadline) { + synchronized (modAckDeadlines) { + modAckDeadlines.add(newAckDeadline); + modAckDeadlines.notifyAll(); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java new file mode 100644 index 00000000..2c98937d --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java @@ -0,0 +1,475 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.pubsub.clients.consumer.ack; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.grpc.FixedExecutorProvider; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.pubsub.clients.consumer.ack.FakeSubscriberServiceImpl.ModifyAckDeadline; +import com.google.pubsub.clients.consumer.ack.Subscriber.Builder; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SubscriberGrpc; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import io.grpc.ManagedChannel; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.internal.ServerImpl; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.threeten.bp.Duration; + +/** Tests for {@link Subscriber}. */ +public class SubscriberTest { + + private static final SubscriptionName TEST_SUBSCRIPTION = + SubscriptionName.create("test-project", "test-subscription"); + + private static final PubsubMessage TEST_MESSAGE = + PubsubMessage.newBuilder().setMessageId("1").build(); + + private static final int INITIAL_ACK_DEADLINE_EXTENSION_SECS = 2; + + private ManagedChannel testChannel; + private FakeScheduledExecutorService fakeExecutor; + private FakeSubscriberServiceImpl fakeSubscriberServiceImpl; + private ServerImpl testServer; + + private TestReceiver testReceiver; + + static class TestReceiver implements MessageReceiver { + private final LinkedBlockingQueue outstandingMessageReplies = + new LinkedBlockingQueue<>(); + private boolean shouldAck = true; // If false, the receiver will nack the messages + private Optional messageCountLatch = Optional.absent(); + private Optional error = Optional.absent(); + private boolean explicitAckReplies; + + void setNackReply() { + this.shouldAck = false; + } + + void setErrorReply(RuntimeException error) { + this.error = Optional.of(error); + } + + void setExplicitAck(boolean explicitAckReplies) { + this.explicitAckReplies = explicitAckReplies; + } + + void setExpectedMessages(int expected) { + this.messageCountLatch = Optional.of(new CountDownLatch(expected)); + } + + @Override + public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) { + try { + if (explicitAckReplies) { + try { + outstandingMessageReplies.put(consumer); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } else { + replyTo(consumer); + } + } finally { + if (messageCountLatch.isPresent()) { + messageCountLatch.get().countDown(); + } + } + } + + void replyAllOutstandingMessage() { + Preconditions.checkState(explicitAckReplies); + AckReplyConsumer reply; + while ((reply = outstandingMessageReplies.poll()) != null) { + replyTo(reply); + } + } + + private void replyTo(AckReplyConsumer reply) { + if (error.isPresent()) { + throw error.get(); + } else { + if (shouldAck) { + reply.ack(); + } else { + reply.nack(); + } + } + } + } + + @Rule public TestName testName = new TestName(); + + @Before + public void setUp() throws Exception { + InProcessServerBuilder serverBuilder = InProcessServerBuilder.forName(testName.getMethodName()); + fakeSubscriberServiceImpl = new FakeSubscriberServiceImpl(); + fakeExecutor = new FakeScheduledExecutorService(); + testChannel = InProcessChannelBuilder.forName(testName.getMethodName()).build(); + serverBuilder.addService(fakeSubscriberServiceImpl); + testServer = serverBuilder.build(); + testServer.start(); + + testReceiver = new TestReceiver(); + } + + @After + public void tearDown() throws Exception { + testServer.shutdownNow().awaitTermination(); + } + + @Test + public void testAckSingleMessage() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + List testAckIds = ImmutableList.of("A"); + + sendMessages(testAckIds); + subscriber.pull(10000); + + subscriber.commit(true); + subscriber.stopAsync(); + + assertEquivalent(testAckIds, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(1)); + } + + @Test + public void testGetSubscriptionZeroTimes() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + sendMessages(ImmutableList.of("A")); + subscriber.pull(10000); + + // Trigger ack sending + subscriber.stopAsync(); + + assertEquals(0, fakeSubscriberServiceImpl.getSubscriptionCalledCount()); + } + + @Test + public void testNackSingleMessage() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + testReceiver.setNackReply(); + sendMessages(ImmutableList.of("A")); + subscriber.pull(10000); + + // Trigger ack sending + subscriber.stopAsync(); + + assertEquivalent( + ImmutableList.of(new ModifyAckDeadline("A", 0)), + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(1)); + } + + @Test + public void testReceiverError_NacksMessage() throws Exception { + testReceiver.setErrorReply(new RuntimeException("Can't process message")); + + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + sendMessages(ImmutableList.of("A")); + subscriber.pull(10000); + + // Trigger nack sending + subscriber.stopAsync().awaitTerminated(); + + assertEquivalent( + ImmutableList.of(new ModifyAckDeadline("A", 0)), + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(1)); + } + + @Test + public void testManualCommitBatchAcks() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + List testAckIdsBatch1 = ImmutableList.of("A", "B", "C"); + sendMessages(testAckIdsBatch1); + subscriber.pull(10000); + + subscriber.commit(true); + + assertEquivalent(testAckIdsBatch1, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(3)); + + // Ensures the next ack sending alarm gets properly setup + List testAckIdsBatch2 = ImmutableList.of("D", "E"); + sendMessages(testAckIdsBatch2); + subscriber.pull(10000); + + subscriber.commit(true); + + assertEquivalent(testAckIdsBatch2, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(2)); + + subscriber.stopAsync().awaitTerminated(); + } + + @Test + public void testAutoCommitBatchAcks() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver).setAutoCommit(true) + .setAutoCommitIntervalMs(6000)); + + sendMessages(ImmutableList.of("R")); + subscriber.pull(10000); + + List testAckIdsBatch1 = ImmutableList.of("A", "B", "C"); + sendMessages(testAckIdsBatch1); + subscriber.pull(10000); + + assertTrue(fakeSubscriberServiceImpl.getAcks().isEmpty()); + + fakeExecutor.advanceTime(Duration.ofSeconds(6)); + + sendMessages(ImmutableList.of("N")); + subscriber.pull(10000); + fakeExecutor.advanceTime(Duration.ofSeconds(6)); + + assertEquivalent(ImmutableList.of("A", "B", "C", "R"), fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(4)); + assertTrue(fakeSubscriberServiceImpl.getAcks().isEmpty()); + + List testAckIdsBatch2 = ImmutableList.of("D", "E"); + sendMessages(testAckIdsBatch2); + fakeExecutor.advanceTime(Duration.ofSeconds(6)); + subscriber.pull(10000); + + assertEquivalent(ImmutableList.of("N"), fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(1)); + assertTrue(fakeSubscriberServiceImpl.getAcks().isEmpty()); + + subscriber.stopAsync().awaitTerminated(); + } + + @Test + public void testBatchAcksAndNacks() throws Exception { + Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); + + // Send messages to be acked + List testAckIdsBatch1 = ImmutableList.of("A", "B", "C"); + sendMessages(testAckIdsBatch1); + subscriber.pull(10000); + + // Send messages to be nacked + List testAckIdsBatch2 = ImmutableList.of("D", "E"); + // Nack messages + testReceiver.setNackReply(); + sendMessages(testAckIdsBatch2); + subscriber.pull(10000); + + subscriber.commit(true); + + subscriber.stopAsync().awaitTerminated(); + + assertEquivalent(testAckIdsBatch1, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(3)); + assertEquivalent( + ImmutableList.of(new ModifyAckDeadline("D", 0), new ModifyAckDeadline("E", 0)), + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(2)); + } + + @Test + public void testModifyAckDeadline() throws Exception { + Subscriber subscriber = + startSubscriber( + getTestSubscriberBuilder(testReceiver) + .setAckExpirationPadding(Duration.ofSeconds(1)) + .setMaxAckExtensionPeriod(13)); + // Send messages to be acked + List testAckIdsBatch = ImmutableList.of("A", "B", "C"); + testReceiver.setExplicitAck(true); + // A modify ack deadline should be scheduled for the next 9s + fakeExecutor.setupScheduleExpectation(Duration.ofSeconds(9)); + sendMessages(testAckIdsBatch); + subscriber.pull(10000); + // To ensure first modify ack deadline got scheduled + fakeExecutor.waitForExpectedWork(); + + fakeExecutor.advanceTime(Duration.ofSeconds(9)); + + assertEquivalentWithTransformation( + testAckIdsBatch, + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), + new Function() { + @Override + public ModifyAckDeadline apply(String ack) { + return new ModifyAckDeadline(ack, INITIAL_ACK_DEADLINE_EXTENSION_SECS); + } + }); + + fakeExecutor.advanceTime(Duration.ofSeconds(1)); + + assertEquivalentWithTransformation( + testAckIdsBatch, + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), + new Function() { + @Override + public ModifyAckDeadline apply(String ack) { + return new ModifyAckDeadline(ack, 3); // It is expected that the deadline is renewed + // only three more seconds to not pass the max + // ack deadline ext. + } + }); + + // No more modify ack deadline extension should be triggered at this point + fakeExecutor.advanceTime(Duration.ofSeconds(20)); + + assertTrue(fakeSubscriberServiceImpl.getModifyAckDeadlines().isEmpty()); + + testReceiver.replyAllOutstandingMessage(); + subscriber.stopAsync().awaitTerminated(); + } + + @Test + public void testModifyAckDeadline_defaultMaxExtensionPeriod() throws Exception { + Subscriber subscriber = + startSubscriber( + getTestSubscriberBuilder(testReceiver) + .setAckExpirationPadding(Duration.ofSeconds(1))); + // Send messages to be acked + List testAckIdsBatch = ImmutableList.of("A", "B", "C"); + testReceiver.setExplicitAck(true); + // A modify ack deadline should be schedule for the next 9s + fakeExecutor.setupScheduleExpectation(Duration.ofSeconds(9)); + sendMessages(testAckIdsBatch); + subscriber.pull(10000); + // To ensure the first modify ack deadlines got scheduled + fakeExecutor.waitForExpectedWork(); + + // Next modify ack deadline should be schedule in the next 1s + fakeExecutor.advanceTime(Duration.ofSeconds(9)); + + assertEquivalentWithTransformation( + testAckIdsBatch, + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), + new Function() { + @Override + public ModifyAckDeadline apply(String ack) { + return new ModifyAckDeadline(ack, INITIAL_ACK_DEADLINE_EXTENSION_SECS); + } + }); + + fakeExecutor.advanceTime(Duration.ofSeconds(1)); + int timeIncrementSecs = INITIAL_ACK_DEADLINE_EXTENSION_SECS; // Second time increment + + // Check ack deadline extensions while the current time has not reached 60 minutes + while (fakeExecutor.getClock().millisTime() + timeIncrementSecs - 1 < 1000 * 60 * 60) { + timeIncrementSecs *= 2; + final int expectedIncrementSecs = Math.min(600, timeIncrementSecs); + assertEquivalentWithTransformation( + testAckIdsBatch, + fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), + new Function() { + @Override + public ModifyAckDeadline apply(String ack) { + return new ModifyAckDeadline(ack, expectedIncrementSecs); + } + }); + fakeExecutor.advanceTime(Duration.ofSeconds(timeIncrementSecs - 1)); + } + + // No more modify ack deadline extension should be triggered at this point + fakeExecutor.advanceTime(Duration.ofSeconds(20)); + + assertTrue(fakeSubscriberServiceImpl.getModifyAckDeadlines().isEmpty()); + + testReceiver.replyAllOutstandingMessage(); + subscriber.stopAsync().awaitTerminated(); + } + + private Subscriber startSubscriber(Builder testSubscriberBuilder) throws Exception { + Subscriber subscriber = testSubscriberBuilder.build(); + subscriber.startAsync().awaitRunning(); + return subscriber; + } + + private void sendMessages(Iterable ackIds) + throws InterruptedException, IOException, ExecutionException { + List messages = new ArrayList(); + for (String ackId : ackIds) { + messages.add(ReceivedMessage.newBuilder().setAckId(ackId).setMessage(TEST_MESSAGE).build()); + } + testReceiver.setExpectedMessages(messages.size()); + fakeSubscriberServiceImpl.enqueuePullResponse( + PullResponse.newBuilder().addAllReceivedMessages(messages).build()); + } + + private Builder getTestSubscriberBuilder(MessageReceiver receiver) { + Subscription subscription = Subscription.newBuilder() + .setName("projects/aaa/subscriptions/bbb") + .setTopic("ccc") + .setAckDeadlineSeconds(10) + .build(); + SubscriberFutureStub subscriberFutureStub = SubscriberGrpc.newFutureStub(testChannel); + return Subscriber.defaultBuilder(subscription, receiver) + .setSystemExecutorProvider(FixedExecutorProvider.create(fakeExecutor)) + .setSubscriberFutureStub(subscriberFutureStub) + .setAutoCommit(false) + .setAutoCommitIntervalMs(5000) + .setMaxAckExtensionPeriod(3600) + .setMaxPerRequestChanges(1000) + .setMaxPullRecords(1000L) + .setAckRequestTimeoutMs(10000) + .setClock(fakeExecutor.getClock()); + } + + @SuppressWarnings("unchecked") + private void assertEquivalent(Collection expectedElems, Collection target) { + List remaining = new ArrayList(target.size()); + remaining.addAll(target); + + for (T expectedElem : expectedElems) { + if (!remaining.contains(expectedElem)) { + throw new AssertionError( + String.format("Expected element %s is not contained in %s", expectedElem, target)); + } + remaining.remove(expectedElem); + } + } + + @SuppressWarnings("unchecked") + private void assertEquivalentWithTransformation( + Collection expectedElems, Collection target, Function transform) { + List remaining = new ArrayList(target.size()); + remaining.addAll(target); + + for (E expectedElem : expectedElems) { + T expected = transform.apply(expectedElem); + if (!remaining.contains(expected)) { + throw new AssertionError( + String.format("Expected element %s is not contained in %s", expected, target)); + } + remaining.remove(expectedElem); + } + } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java index 39de8019..5c5fb426 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java @@ -16,56 +16,46 @@ package com.google.pubsub.clients.producer; -import org.junit.Test; -import org.junit.Before; -import org.junit.Assert; -import org.junit.runner.RunWith; - -import org.mockito.Matchers; -import org.mockito.ArgumentCaptor; - -import com.google.pubsub.v1.TopicName; -import com.google.pubsub.v1.PubsubMessage; +import static org.powermock.api.mockito.PowerMockito.when; +import com.google.api.core.ApiFuture; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.retrying.RetrySettings; import com.google.cloud.pubsub.v1.Publisher; -import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.pubsub.v1.Publisher.Builder; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.api.gax.batching.BatchingSettings; - +import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.common.collect.ImmutableMap; - +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.TopicName; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerInterceptor; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.ProducerInterceptor; - -import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.IntegerDeserializer; - -import java.io.PrintStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.ByteArrayOutputStream; - -import java.util.Map; -import java.util.List; -import java.util.ArrayList; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import com.google.api.core.ApiFuture; - +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; -import org.powermock.modules.junit4.PowerMockRunner; -import static org.powermock.api.mockito.PowerMockito.when; import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PrepareForTest({Publisher.class, TopicAdminClient.class}) From f1bbb52527e6a5148944cc2610e1005f5366b4ea Mon Sep 17 00:00:00 2001 From: pietrzykp Date: Fri, 29 Sep 2017 16:58:22 -0400 Subject: [PATCH 135/140] Commit kept messages before offset (#127) * Commit messages with offset before param --- .../clients/consumer/KafkaConsumer.java | 48 +++++++--- .../consumer/ack/MessageDispatcher.java | 32 ++++++- .../ack/PollingSubscriberConnection.java | 5 +- .../clients/consumer/ack/Subscriber.java | 6 +- .../clients/consumer/KafkaConsumerTest.java | 94 ++++++++++++++++++- 5 files changed, 164 insertions(+), 21 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index b0096c0d..100d5fd4 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -93,6 +93,7 @@ public class KafkaConsumer implements Consumer { private static final int DEFAULT_CHECKSUM = 1; private static final String KEY_ATTRIBUTE = "key"; + private static final String OFFSET_ATTRIBUTE = "offset"; private final Config config; private final SubscriberFutureStub subscriberFutureStub; @@ -180,7 +181,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener liste Map subscriptionMap = getSubscriptionsFromPubsub(futureSubscriptions); Map tempSubscribersMap = new HashMap<>(); - for(Map.Entry entry: subscriptionMap.entrySet()) { + for(Map.Entry entry : subscriptionMap.entrySet()) { Subscriber subscriber = getSubscriberFromConfigs(entry); tempSubscribersMap.put(entry.getKey(), subscriber); @@ -213,7 +214,7 @@ private List> deputePubsubSubscribesGet(Collection> responseDatas = new ArrayList<>(); Set usedNames = new HashSet<>(); - for (String topic: topics) { + for (String topic : topics) { if (!usedNames.contains(topic)) { String subscriptionString = SUBSCRIPTION_PREFIX + topic + "_" + config.getGroupId(); ListenableFuture deputedSubscription = @@ -237,7 +238,7 @@ private Map getSubscriptionsFromPubsub( List> responseDatas) { Map subscriptionMap = new HashMap<>(); - for (ResponseData responseData: responseDatas) { + for (ResponseData responseData : responseDatas) { boolean success = false; try { Subscription s = responseData.getRequestListenableFuture().get(); @@ -296,7 +297,7 @@ private void checkSubscribePreconditions(Collection topics) { Preconditions.checkArgument(topics != null, "Topic collection to subscribe to cannot be null"); - for (String topic: topics) { + for (String topic : topics) { Preconditions.checkArgument(topic != null && !topic.trim().isEmpty(), "Topic collection to subscribe to cannot contain null or empty topic"); } @@ -319,7 +320,7 @@ public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { List matchingTopics = new ArrayList<>(); - for (Topic topic: existingTopics) { + for (Topic topic : existingTopics) { String topicName = topic.getName().substring(TOPIC_PREFIX.length(), topic.getName().length()); Matcher m = pattern.matcher(topicName); if (m.matches()) @@ -352,7 +353,7 @@ private void checkPatternSubscribePreconditions(Pattern pattern) { @Override public void unsubscribe() { - for(Subscriber s: topicNameToSubscriber.values()) { + for(Subscriber s : topicNameToSubscriber.values()) { s.stopAsync().awaitTerminated(); } @@ -366,7 +367,7 @@ public void unsubscribe() { private List getSubscriptionsFromSubcribers() { List subscriptions = new ArrayList<>(topicNameToSubscriber.size()); - for(Subscriber s: topicNameToSubscriber.values()) { + for(Subscriber s : topicNameToSubscriber.values()) { subscriptions.add(s.getSubscription()); } return subscriptions; @@ -377,7 +378,7 @@ private void deleteSubscriptionsIfAllowed(Collection subscriptions return; List> listenableFutures = new ArrayList<>(); - for (Subscription s: subscriptions) { + for (Subscription s : subscriptions) { ListenableFuture emptyListenableFuture = subscriberFutureStub .deleteSubscription(DeleteSubscriptionRequest.newBuilder() .setSubscription(s.getName()).build()); @@ -453,11 +454,17 @@ private void checkPollPreconditions(long timeout) { private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, String topic) { PubsubMessage message = receivedMessage.getMessage(); - long timestamp = message.getPublishTime().getSeconds(); + long timestamp = message.getPublishTime().getSeconds() * 1000 + message.getPublishTime().getNanos() / 1000; TimestampType timestampType = TimestampType.CREATE_TIME; //because of no offset concept in PubSub, timestamp is treated as an offset - long offset = timestamp; + String offsetString = message.getAttributesOrDefault(OFFSET_ATTRIBUTE, "0"); + long offset; + try { + offset = Long.parseLong(offsetString); + } catch (NumberFormatException e) { + throw new KafkaException("Offset attribute in message in not parsable", e); + } //key of Kafka-style message is stored in PubSub attributes (null possible) String key = message.getAttributesOrDefault(KEY_ATTRIBUTE, null); @@ -488,7 +495,7 @@ public void commitSync() { @Override public void commitSync(final Map offsets) { - throw new UnsupportedOperationException("Not yet implemented"); + commitForTopicAndOffset(offsets, true); } @Override @@ -498,13 +505,15 @@ public void commitAsync() { @Override public void commitAsync(OffsetCommitCallback callback) { - throw new UnsupportedOperationException("Not yet implemented"); + log.warn("OffsetCommitCallback is not supported and will not be invoked"); + commitAsync(); } @Override public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - throw new UnsupportedOperationException("Not yet implemented"); + log.warn("OffsetCommitCallback is not supported and will not be invoked"); + commitForTopicAndOffset(offsets, false); } private void commit(boolean sync) { @@ -513,6 +522,19 @@ private void commit(boolean sync) { } } + private void commitForTopicAndOffset(Map offsets, boolean sync) { + for(Entry commitOffsets : offsets.entrySet()) { + String topic = commitOffsets.getKey().topic(); + long offset = commitOffsets.getValue().offset(); + Subscriber subscriber = topicNameToSubscriber.get(topic); + if(subscriber != null) { + subscriber.commitBefore(sync, offset); + } else { + log.warn("Topic {} is not subscribed to", topic); + } + } + } + @Override public void seek(TopicPartition partition, long offset) { throw new UnsupportedOperationException("Not yet implemented"); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java index c55ca83b..18adc70f 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/MessageDispatcher.java @@ -34,6 +34,7 @@ import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -176,13 +177,15 @@ public enum AckReply { */ private class AckHandler implements FutureCallback { private final String ackId; + private final long offset; private final int outstandingBytes; private final AtomicBoolean acked; private final long receivedTimeMillis; - AckHandler(String ackId, int outstandingBytes) { + AckHandler(String ackId, int outstandingBytes, long offset) { this.ackId = ackId; this.outstandingBytes = outstandingBytes; + this.offset = offset; acked = new AtomicBoolean(false); receivedTimeMillis = clock.millisTime(); } @@ -327,8 +330,17 @@ void processReceivedMessages(List messages, Runnable doneCallba OutstandingMessagesBatch outstandingBatch = new OutstandingMessagesBatch(doneCallback); final ArrayList ackHandlers = new ArrayList<>(messages.size()); for (ReceivedMessage message : messages) { + long offset; + + try { + offset = Long.parseLong(message.getMessage().getAttributesOrDefault("offset", "0")); + } catch (NumberFormatException e) { + throw new KafkaException("Offset attribute in message in not parsable", e); + } + AckHandler ackHandler = - new AckHandler(message.getAckId(), message.getMessage().getSerializedSize()); + new AckHandler(message.getAckId(), message.getMessage().getSerializedSize(), offset); + ackHandlers.add(ackHandler); outstandingBatch.addMessage(message, ackHandler); } @@ -530,7 +542,7 @@ private void setupNextAckDeadlineExtensionAlarm(Instant expiration) { } } - void acknowledgePendingMessages(boolean sync) { + void acknowledgePendingMessages(boolean sync, Long offset) { Map acksToSend = new HashMap<>(); synchronized (pendingAcks) { if (!pendingAcks.isEmpty()) { @@ -541,6 +553,10 @@ void acknowledgePendingMessages(boolean sync) { } } + if(offset != null) { + filterAcksBeforeOffset(offset, acksToSend); + } + if(sync) { commitSync(acksToSend); } else { @@ -548,6 +564,16 @@ void acknowledgePendingMessages(boolean sync) { } } + private void filterAcksBeforeOffset(Long offset, Map acksToSend) { + Iterator> iter = acksToSend.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + if(entry.getValue().offset > offset) { + iter.remove(); + } + } + } + private void commitAsync(Map acksToSend) { ListenableFuture> listListenableFuture = ackProcessor .sendAckOperations(new ArrayList<>(acksToSend.keySet()), Collections.emptyList()); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java index b9ee10c1..6488c830 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/PollingSubscriberConnection.java @@ -37,7 +37,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.KafkaException; import org.threeten.bp.Duration; @@ -115,8 +114,8 @@ private boolean isAlive() { return state == State.RUNNING || state == State.STARTING; } - void commit(boolean sync, OffsetCommitCallback callback) { - messageDispatcher.acknowledgePendingMessages(sync); + void commit(boolean sync, Long offset) { + messageDispatcher.acknowledgePendingMessages(sync, offset); } @Override diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java index 5eaf17ad..d97e4923 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java @@ -203,8 +203,12 @@ public ApiService startAsync() { return super.startAsync(); } + public void commitBefore(boolean sync, Long offset) { + pollingSubscriberConnection.commit(sync, offset); + } + public void commit(boolean sync) { - pollingSubscriberConnection.commit(sync, null); + commitBefore(sync, null); } diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java index d0eb3ed9..fa3aeb07 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -43,15 +43,20 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.Rule; @@ -265,6 +270,29 @@ public void pollExecutionException() throws ExecutionException, InterruptedExcep } } + @Test + public void commitBefore() { + KafkaConsumer consumer = getConsumer(false); + SubscriberCommitOffsetImpl subscriberCommitOffsetService = new SubscriberCommitOffsetImpl(); + grpcServerRule.getServiceRegistry().addService(subscriberCommitOffsetService); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + + for(int i = 0; i < 10; ++i) { + consumer.poll(1000); + } + + assertEquals(10, subscriberCommitOffsetService.getNotAcknowledgedSize()); + + Map topicOffsets = new HashMap<>(); + topicOffsets.put(new TopicPartition(topic, 0), new OffsetAndMetadata(6)); + + consumer.commitSync(topicOffsets); + + assertEquals(3, subscriberCommitOffsetService.getNotAcknowledgedSize()); + } + private KafkaConsumer getConsumer(boolean allowesCreation) { Properties properties = getTestProperties(allowesCreation); Config configOptions = new Config(properties); @@ -280,6 +308,7 @@ private Properties getTestProperties(boolean allowesCreation) { "org.apache.kafka.common.serialization.StringDeserializer") .put("max.poll.records", 500) .put("group.id", "groupId") + .put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false) .put("subscription.allow.create", allowesCreation) .put("subscription.allow.delete", false) .build() @@ -293,6 +322,9 @@ private String getTopicPrefixString(String topicName) { static class SubscriberGetImpl extends SubscriberImplBase { + private Timestamp keptTimestamp = Timestamp.newBuilder().setSeconds(1500).build(); + private Long keptOffset = 1234567891234L; + @Override public void createSubscription(Subscription request, StreamObserver responseObserver) { responseObserver.onError(new Throwable("This subscriber does not let creating subscriptions")); @@ -322,8 +354,9 @@ public void pull(PullRequest request, StreamObserver responseObser Timestamp timestamp = Timestamp.newBuilder().setSeconds(1500).build(); PubsubMessage pubsubMessage = PubsubMessage.newBuilder() - .setPublishTime(timestamp) + .setPublishTime(keptTimestamp) .putAttributes("key", serializedKey) + .putAttributes("offset", keptOffset.toString()) .setData(ByteString.copyFrom(serializedValueBytes)) .build(); @@ -346,6 +379,65 @@ public void acknowledge(AcknowledgeRequest request, StreamObserver respon } } + static class SubscriberCommitOffsetImpl extends SubscriberImplBase { + + long currentOffset = 0; + Set pulledMessagesOffsets = new HashSet<>(); + + @Override + public void getSubscription(GetSubscriptionRequest request, StreamObserver responseObserver) { + Subscription s = Subscription.newBuilder().setName("projects/null/subscriptions/name").setTopic("projects/null/topics/topic").build(); + responseObserver.onNext(s); + responseObserver.onCompleted(); + } + + @Override + public void pull(PullRequest request, StreamObserver responseObserver) { + String topic = "topic"; + Integer key = 125; + String value = "value"; + byte[] serializedKeyBytes = new IntegerSerializer().serialize(topic, key); + String serializedKey = new String(Base64.encodeBase64(serializedKeyBytes)); + byte[] serializedValueBytes = new StringSerializer().serialize(topic, value); + + PubsubMessage pubsubMessage = PubsubMessage.newBuilder() + .putAttributes("key", serializedKey) + .putAttributes("offset", Long.toString(currentOffset)) + .setData(ByteString.copyFrom(serializedValueBytes)) + .setMessageId(Long.toString(currentOffset)) + .build(); + + pulledMessagesOffsets.add(Long.toString(currentOffset)); + + ReceivedMessage receivedMessage = ReceivedMessage.newBuilder() + .setMessage(pubsubMessage) + .setAckId(Long.toString(currentOffset)) + .build(); + + PullResponse pullResponse = PullResponse.newBuilder() + .addReceivedMessages(receivedMessage) + .build(); + + currentOffset++; + + responseObserver.onNext(pullResponse); + responseObserver.onCompleted(); + } + + @Override + public void acknowledge(AcknowledgeRequest request, StreamObserver responseObserver) { + for(String ackId: request.getAckIdsList()) { + pulledMessagesOffsets.remove(ackId); + } + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + + public long getNotAcknowledgedSize() { + return pulledMessagesOffsets.size(); + } + } + static class SubscriberNoExistingSubscriptionsImpl extends SubscriberImplBase { @Override public void createSubscription(Subscription request, StreamObserver responseObserver) { From 8151fc0e11d7acb83133c8923af508e669d8e353 Mon Sep 17 00:00:00 2001 From: pietrzykp Date: Fri, 29 Sep 2017 17:13:24 -0400 Subject: [PATCH 136/140] Mapped api seek and other functionalities (#126) * Seek (variations) * Assign & assignment * Other methods * Interceptors --- .../pubsub/clients/consumer/Config.java | 29 +++ .../clients/consumer/KafkaConsumer.java | 196 ++++++++++++++--- .../clients/consumer/KafkaConsumerTest.java | 199 +++++++++++++++++- .../consumer/TestConsumerInterceptor.java | 65 ++++++ .../clients/consumer/ack/SubscriberTest.java | 5 +- 5 files changed, 463 insertions(+), 31 deletions(-) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/TestConsumerInterceptor.java diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java index 23bd2def..74b50945 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java @@ -15,10 +15,14 @@ package com.google.pubsub.clients.consumer; import com.google.common.base.Preconditions; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerConfigCreator; +import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; import org.apache.kafka.common.serialization.Deserializer; class Config { @@ -37,6 +41,8 @@ class Config { private final Integer createdSubscriptionDeadlineSeconds; private final Integer requestTimeoutMs; private final Integer maxAckExtensionPeriod; + private final ConsumerInterceptors interceptors; + private static final AtomicInteger CLIENT_ID = new AtomicInteger(1); Config(Map configs) { @@ -84,6 +90,25 @@ private Config(ConsumerConfig consumerConfig, ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer, true); this.valueDeserializer = handleDeserializer(consumerConfig, ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer, false); + + String id = consumerConfig.getString(ConsumerConfig.CLIENT_ID_CONFIG); + String clientId = ""; + if (id.length() <= 0) { + clientId = "consumer-" + CLIENT_ID.getAndIncrement(); + } else { + clientId = id; + } + + consumerConfig.originals().put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + + List interceptorList = + (ConsumerConfigCreator + .getConsumerConfig(consumerConfig.originals())) + .getConfiguredInstances( + ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptor.class); + + this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); + this.groupId = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG); //this is a limit on each poll for each topic this.maxPollRecords = consumerConfig.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); @@ -162,6 +187,10 @@ Integer getMaxAckExtensionPeriod() { return maxAckExtensionPeriod; } + ConsumerInterceptors getInterceptors() { + return interceptors; + } + private Deserializer handleDeserializer(ConsumerConfig configs, String configString, Deserializer providedDeserializer, boolean isKey) { Deserializer deserializer; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index 100d5fd4..c284bdc3 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -24,6 +24,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.Empty; +import com.google.protobuf.Timestamp; import com.google.pubsub.clients.consumer.ack.MappedApiMessageReceiver; import com.google.pubsub.clients.consumer.ack.Subscriber; import com.google.pubsub.common.ChannelUtil; @@ -36,6 +37,8 @@ import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SeekRequest; +import com.google.pubsub.v1.SeekResponse; import com.google.pubsub.v1.SubscriberGrpc; import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; import com.google.pubsub.v1.Subscription; @@ -46,7 +49,9 @@ import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -68,15 +73,22 @@ import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.utils.Utils; +import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * This class, as Kafka's KafkaConsumer, IS NOT THREAD SAFE. + * @param Key + * @param value + */ public class KafkaConsumer implements Consumer { private static final ConsumerRebalanceListener NO_REBALANCE_LISTENER = null; @@ -100,8 +112,14 @@ public class KafkaConsumer implements Consumer { private final PublisherFutureStub publisherFutureStub; private ImmutableList topicNames = ImmutableList.of(); - private ImmutableMap topicNameToSubscriber = ImmutableMap.of(); + private Set pausedTopics = new HashSet<>(); + private Map lazySeeks = new HashMap<>(); + + enum Seek { + BEGINNING, + END + } private int currentPoolIndex; @@ -158,7 +176,11 @@ private KafkaConsumer(Config configOptions) { @Override public Set assignment() { - throw new UnsupportedOperationException("Not yet implemented"); + Set partitions = new HashSet<>(); + for(String topicName: topicNames) { + partitions.add(new TopicPartition(topicName, DEFAULT_PARTITION)); + } + return partitions; } @Override @@ -362,6 +384,8 @@ public void unsubscribe() { topicNameToSubscriber = ImmutableMap.of(); topicNames = ImmutableList.of(); + pausedTopics = new HashSet<>(); + lazySeeks = new HashMap<>(); currentPoolIndex = 0; } @@ -393,21 +417,29 @@ private void deleteSubscriptionsIfAllowed(Collection subscriptions public ConsumerRecords poll(long timeout) { checkPollPreconditions(timeout); + if(!lazySeeks.isEmpty()) { + performLazySeekCalls(); + } + int startedAtIndex = this.currentPoolIndex; + ConsumerRecords consumerRecords = new ConsumerRecords<>(new HashMap<>()); try { do { - + //TODO if multiple pulls are done here, timeout should be split among them. String topicName = topicNames.get(this.currentPoolIndex % topicNameToSubscriber.size()); - Subscriber subscriber = topicNameToSubscriber.get(topicName); + if(!pausedTopics.contains(topicName)) { + Subscriber subscriber = topicNameToSubscriber.get(topicName); + PullResponse pullResponse = subscriber.pull(timeout); - PullResponse pullResponse = subscriber.pull(timeout); + List> subscriptionRecords = mapToConsumerRecords(topicName, pullResponse); - List> subscriptionRecords = mapToConsumerRecords(topicName, pullResponse); - this.currentPoolIndex = (this.currentPoolIndex + 1) % topicNameToSubscriber.size(); - - if (!pullResponse.getReceivedMessagesList().isEmpty()) { - return getConsumerRecords(topicName, subscriptionRecords); + if (!pullResponse.getReceivedMessagesList().isEmpty()) { + consumerRecords = getConsumerRecords(topicName, subscriptionRecords); + incrementPollIndex(); + break; + } } + incrementPollIndex(); } while (this.currentPoolIndex != startedAtIndex); } catch (InterruptedException e) { @@ -416,7 +448,14 @@ public ConsumerRecords poll(long timeout) { throw new KafkaException(e); } - return new ConsumerRecords<>(new HashMap<>()); + if (config.getInterceptors() != null) { + consumerRecords = config.getInterceptors().onConsume(consumerRecords); + } + return consumerRecords; + } + + private void incrementPollIndex() { + this.currentPoolIndex = (this.currentPoolIndex + 1) % topicNameToSubscriber.size(); } private ConsumerRecords getConsumerRecords(String topicName, @@ -455,6 +494,7 @@ private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, PubsubMessage message = receivedMessage.getMessage(); long timestamp = message.getPublishTime().getSeconds() * 1000 + message.getPublishTime().getNanos() / 1000; + TimestampType timestampType = TimestampType.CREATE_TIME; //because of no offset concept in PubSub, timestamp is treated as an offset @@ -485,7 +525,11 @@ private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, @Override public void assign(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + Set topics = new HashSet<>(); + for(TopicPartition topicPartition: partitions) { + topics.add(topicPartition.topic()); + } + subscribe(topics); } @Override @@ -520,6 +564,11 @@ private void commit(boolean sync) { for (Map.Entry entry : topicNameToSubscriber.entrySet()) { entry.getValue().commit(sync); } + + /*We have no concept of offsets, so it is impossible to pass an argument to the interceptor that + makes any sense. If to be resolved in the future, the call for sync call should go somewhere here.*/ + /*if (sync && interceptors != null) + interceptors.onCommit(offsets);*/ } private void commitForTopicAndOffset(Map offsets, boolean sync) { @@ -537,73 +586,164 @@ private void commitForTopicAndOffset(Map offs @Override public void seek(TopicPartition partition, long offset) { - throw new UnsupportedOperationException("Not yet implemented"); + seekToTimestamp(Collections.singletonList(partition.topic()), offset); } @Override public void seekToBeginning(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + seekToPlace(partitions, Seek.BEGINNING); } @Override public void seekToEnd(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + seekToPlace(partitions, Seek.END); + } + + private void seekToPlace(Collection partitions, Seek seek) { + if(partitions == null || partitions.isEmpty()) { + for(String topic: topicNames) { + lazySeeks.put(topic, seek); + } + } else { + for(TopicPartition partition: partitions) { + lazySeeks.put(partition.topic(), seek); + } + } + } + + private void performLazySeekCalls() { + List beginningTopics = new ArrayList<>(); + List endTopics = new ArrayList<>(); + + for(Map.Entry entry: lazySeeks.entrySet()) { + if (Seek.BEGINNING.equals(entry.getValue())) { + beginningTopics.add(entry.getKey()); + } else { + endTopics.add(entry.getKey()); + } + } + + seekToTimestamp(beginningTopics, 0); + + long now = new DateTime().getMillis(); + seekToTimestamp(endTopics, now); + + lazySeeks = new HashMap<>(); + } + + private void seekToTimestamp(Collection topics, long timestamp) { + Timestamp protobufTimestamp = Timestamp.newBuilder().setSeconds(timestamp / 1000) + .setNanos((int) ((timestamp % 1000) * 1000000)).build(); + + List> seekResponses = new ArrayList<>(); + for(String topic: topics) { + String topicSubscription = topicNameToSubscriber.get(topic).getSubscription().getName(); + ListenableFuture seek = subscriberFutureStub.seek( + SeekRequest.newBuilder() + .setSubscription(topicSubscription) + .setTime(protobufTimestamp) + .build() + ); + seekResponses.add(seek); + } + + ListenableFuture> listListenableFuture = Futures.allAsList(seekResponses); + + try { + listListenableFuture.get(); + } catch (InterruptedException e) { + throw new InterruptException(e); + } catch (ExecutionException e) { + throw new KafkaException(e); + } } @Override public long position(TopicPartition partition) { - throw new UnsupportedOperationException("Not yet implemented"); + throw new UnsupportedOperationException("This method has no mapping in PubSub"); } @Override public OffsetAndMetadata committed(TopicPartition partition) { - throw new UnsupportedOperationException("Not yet implemented"); + throw new UnsupportedOperationException("This method has no mapping in PubSub "); } @Override public Map metrics() { - throw new UnsupportedOperationException("Not yet implemented"); + return new HashMap<>(); } @Override public List partitionsFor(String topic) { - throw new UnsupportedOperationException("Not yet implemented"); + Node[] dummy = {new Node(0, "", 0)}; + return Arrays.asList(new PartitionInfo(topic, DEFAULT_PARTITION, dummy[0], dummy, dummy)); } @Override public Map> listTopics() { - throw new UnsupportedOperationException("Not yet implemented"); + Map> partitionTopicMap = new HashMap<>(); + + Node[] dummy = {new Node(0, "", 0)}; + List existingTopics = getPubsubExistingTopics(); + for(Topic topic: existingTopics) { + partitionTopicMap.put(topic.getName(), + Arrays.asList(new PartitionInfo(topic.getName(), DEFAULT_PARTITION, dummy[0], dummy, dummy))); + } + return partitionTopicMap; } @Override public void pause(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + for(TopicPartition partition: partitions) { + pausedTopics.add(partition.topic()); + } } @Override public void resume(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + for(TopicPartition partition: partitions) { + pausedTopics.remove(partition.topic()); + } } @Override public Set paused() { - throw new UnsupportedOperationException("Not yet implemented"); + Set paused = new HashSet<>(); + for(String topic: pausedTopics) { + paused.add(new TopicPartition(topic, DEFAULT_PARTITION)); + } + return paused; } @Override public Map offsetsForTimes( Map timestampsToSearch) { - throw new UnsupportedOperationException("Not yet implemented"); + Map offsetsForTimes = new HashMap<>(); + for(Map.Entry entry: timestampsToSearch.entrySet()) { + Long timestamp = entry.getValue(); + OffsetAndTimestamp offsetAndTimestamp = new OffsetAndTimestamp(timestamp, timestamp); + offsetsForTimes.put(entry.getKey(), offsetAndTimestamp); + } + return offsetsForTimes; } @Override public Map beginningOffsets(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + return getTopicPartitionOffsetMap(partitions, 0L); } @Override public Map endOffsets(Collection partitions) { - throw new UnsupportedOperationException("Not yet implemented"); + long millis = new DateTime().getMillis(); + return getTopicPartitionOffsetMap(partitions, millis); + } + + private Map getTopicPartitionOffsetMap(Collection partitions, long offset) { + Map beginningOffsets = new HashMap<>(); + for(TopicPartition topicPartition: partitions) { + beginningOffsets.put(topicPartition, offset); + } + return beginningOffsets; } /** @@ -617,12 +757,14 @@ public void close() { unsubscribe(); config.getKeyDeserializer().close(); config.getValueDeserializer().close(); + if(config.getInterceptors() != null) + config.getInterceptors().close(); log.debug("PubSub subscriber has been closed"); } @Override public void wakeup() { - throw new UnsupportedOperationException("Not yet implemented"); + throw new UnsupportedOperationException("This method has no mapping in PubSub"); } class ResponseData { diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java index fa3aeb07..a848aa62 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -34,6 +34,8 @@ import com.google.pubsub.v1.PullRequest; import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SeekRequest; +import com.google.pubsub.v1.SeekResponse; import com.google.pubsub.v1.SubscriberGrpc.SubscriberImplBase; import com.google.pubsub.v1.Subscription; import io.grpc.Status; @@ -45,17 +47,23 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.StringSerializer; @@ -293,6 +301,187 @@ public void commitBefore() { assertEquals(3, subscriberCommitOffsetService.getNotAcknowledgedSize()); } + @Test + public void assignment() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + Set topics = new HashSet<>(Arrays.asList("topic1", "topic2")); + consumer.assign(Arrays.asList(new TopicPartition("topic2", 0), new TopicPartition("topic1", 0))); + + Set assignment = consumer.assignment(); + Set partitionTopics = assignment.stream().map(TopicPartition::topic).collect(Collectors.toSet()); + List partitionOffsets = assignment.stream().map(TopicPartition::partition).collect(Collectors.toList()); + + assertEquals(topics, partitionTopics); + assertEquals(Arrays.asList(0, 0), partitionOffsets); + consumer.close(); + } + + @Test + public void seekTimestamp() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + + long offset = 12345678000L; + consumer.seek(new TopicPartition(topic, 0), offset); + ConsumerRecords poll = consumer.poll(1000); + + for(ConsumerRecord record : poll.records(topic)) { + assertEquals(record.offset(), offset); + } + } + + @Test + public void seekToBeginning() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + consumer.seekToBeginning(Collections.singletonList(new TopicPartition(topic, 0))); + + ConsumerRecords poll = consumer.poll(1000); + + for(ConsumerRecord record : poll.records(topic)) { + assertEquals(record.offset(), 0); + } + } + + @Test + public void seekToBeginningEmptyList() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + consumer.seekToBeginning(new ArrayList<>()); + + ConsumerRecords poll = consumer.poll(1000); + + for(ConsumerRecord record : poll.records(topic)) { + assertEquals(record.offset(), 0); + } + } + + @Test + public void checkBeginningOffsets() { + KafkaConsumer consumer = getConsumer(false); + List partitions = Arrays.asList(new TopicPartition("topic1", 0), + new TopicPartition("topic2", 0)); + Map offsetsMap = consumer.beginningOffsets(partitions); + for(Long offset: offsetsMap.values()) { + assertEquals(new Long(0), offset); + } + } + + @Test + public void partitionsForTopic() { + KafkaConsumer consumer = getConsumer(false); + List topic1 = consumer.partitionsFor("topic1"); + assertEquals(1, topic1.size()); + assertEquals("topic1", topic1.get(0).topic()); + assertEquals(0, topic1.get(0).partition()); + } + + @Test + public void partitionsForAllTopics() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new PublisherImpl()); + + Map> topicPartitionsMap = consumer.listTopics(); + assertEquals(4, topicPartitionsMap.keySet().size()); + for(List partitions: topicPartitionsMap.values()) { + assertEquals(1, partitions.size()); + assertEquals(0, partitions.get(0).partition()); + } + } + + @Test + public void paused() { + KafkaConsumer consumer = getConsumer(false); + List topicPartitions = Arrays.asList(new TopicPartition("topic1", 0), + new TopicPartition("topic2", 0)); + + consumer.pause(topicPartitions); + assertEquals(2, consumer.paused().size()); + + consumer.pause(topicPartitions); + assertEquals(2, consumer.paused().size()); + + consumer.resume(Collections.singletonList(new TopicPartition("topic1", 0))); + assertEquals(1, consumer.paused().size()); + + consumer.resume(Collections.singletonList(new TopicPartition("topic2", 0))); + assertEquals(0, consumer.paused().size()); + } + + @Test + public void pausedDoesNotPoll() { + KafkaConsumer consumer = getConsumer(false); + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + String topic = "topic"; + consumer.subscribe(Collections.singletonList(topic)); + ConsumerRecords poll = consumer.poll(1000); + assertEquals(1, poll.count()); + + consumer.pause(Collections.singletonList(new TopicPartition("topic", 0))); + poll = consumer.poll(1000); + assertEquals(0, poll.count()); + + consumer.resume(Collections.singletonList(new TopicPartition("topic", 0))); + poll = consumer.poll(1000); + assertEquals(1, poll.count()); + } + + @Test + public void offsetsForTimes() { + KafkaConsumer consumer = getConsumer(false); + long millis = 150000000000L; + TopicPartition topicPartition = new TopicPartition("topic", 0); + Map timestampsToSearch = new HashMap<>(); + + timestampsToSearch.put(topicPartition, millis); + + Map result = consumer.offsetsForTimes(timestampsToSearch); + assertTrue(result.containsKey(topicPartition)); + assertEquals(millis, result.get(topicPartition).offset()); + } + + @Test + public void interceptorsOnConsume() { + grpcServerRule.getServiceRegistry().addService(new SubscriberGetImpl()); + + Properties properties = new Properties(); + properties.putAll(new ImmutableMap.Builder<>() + .put("key.deserializer", + "org.apache.kafka.common.serialization.IntegerDeserializer") + .put("value.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer") + .put("max.poll.records", 500) + .put("group.id", "groupId") + .put("subscription.allow.create", false) + .put("subscription.allow.delete", false) + .put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, "com.google.pubsub.clients.consumer.TestConsumerInterceptor") + .build() + ); + Config configOptions = new Config(properties); + KafkaConsumer consumer = new KafkaConsumer<>(configOptions, grpcServerRule.getChannel(), null); + + consumer.subscribe(Collections.singletonList("topic")); + ConsumerRecords poll = consumer.poll(1000); + Iterator> iterator = poll.iterator(); + while(iterator.hasNext()) { + ConsumerRecord next = iterator.next(); + assertEquals(new Integer(-1), next.key()); + assertEquals("_consumed_", next.value()); + } + } + private KafkaConsumer getConsumer(boolean allowesCreation) { Properties properties = getTestProperties(allowesCreation); Config configOptions = new Config(properties); @@ -351,8 +540,6 @@ public void pull(PullRequest request, StreamObserver responseObser String serializedKey = new String(Base64.encodeBase64(serializedKeyBytes)); byte[] serializedValueBytes = new StringSerializer().serialize(topic, value); - Timestamp timestamp = Timestamp.newBuilder().setSeconds(1500).build(); - PubsubMessage pubsubMessage = PubsubMessage.newBuilder() .setPublishTime(keptTimestamp) .putAttributes("key", serializedKey) @@ -377,6 +564,14 @@ public void acknowledge(AcknowledgeRequest request, StreamObserver respon responseObserver.onNext(Empty.getDefaultInstance()); responseObserver.onCompleted(); } + + @Override + public void seek(SeekRequest request, StreamObserver responseObserver) { + this.keptOffset = request.getTime().getSeconds() * 1000 + request.getTime().getNanos() / 1000; + responseObserver.onNext(SeekResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + } static class SubscriberCommitOffsetImpl extends SubscriberImplBase { diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/TestConsumerInterceptor.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/TestConsumerInterceptor.java new file mode 100644 index 00000000..04579f80 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/TestConsumerInterceptor.java @@ -0,0 +1,65 @@ +/* Copyright 2017 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +package com.google.pubsub.clients.consumer; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; + +public class TestConsumerInterceptor implements ConsumerInterceptor { + + @Override + public ConsumerRecords onConsume(ConsumerRecords consumerRecords) { + Map>> pollRecords = new HashMap<>(); + + for(TopicPartition topicPartition: consumerRecords.partitions()) { + Iterable> records = consumerRecords.records(topicPartition.topic()); + List> consumedRecords = new ArrayList<>(); + for(ConsumerRecord next: records) { + + ConsumerRecord newRecord = new ConsumerRecord<>(next.topic(), next.partition(), + next.offset(), next.timestamp(), next.timestampType(), + next.checksum(), next.serializedKeySize(), next.serializedValueSize(), -1, + "_consumed_"); + + consumedRecords.add(newRecord); + } + pollRecords.put(topicPartition, consumedRecords); + } + + return new ConsumerRecords<>(pollRecords); + } + + @Override + public void onCommit(Map map) { + + } + + @Override + public void close() { + + } + + @Override + public void configure(Map map) { + + } +} diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java index 2c98937d..2311b60c 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java @@ -266,8 +266,9 @@ public void testAutoCommitBatchAcks() throws Exception { fakeExecutor.advanceTime(Duration.ofSeconds(6)); subscriber.pull(10000); - assertEquivalent(ImmutableList.of("N"), fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(1)); - assertTrue(fakeSubscriberServiceImpl.getAcks().isEmpty()); + List acks = fakeSubscriberServiceImpl.getAcks(); + assertTrue(!acks.contains("D")); + assertTrue(!acks.contains("E")); subscriber.stopAsync().awaitTerminated(); } From 8a0bdf84f209cae4fcb2927e57e1ae3c017605ac Mon Sep 17 00:00:00 2001 From: pietrzykp Date: Fri, 29 Sep 2017 18:13:30 -0400 Subject: [PATCH 137/140] Mapped api consumer javadoc (#128) --- .../clients/consumer/KafkaConsumer.java | 159 +++++++++++++++++- 1 file changed, 151 insertions(+), 8 deletions(-) diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index c284bdc3..ca1837c0 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -60,6 +60,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -86,6 +87,24 @@ /** * This class, as Kafka's KafkaConsumer, IS NOT THREAD SAFE. + * You must specify Google Cloud project in env variable GOOGLE_CLOUD_PROJECT. + * In this implementation, due to differences in Kafka and Pub/Sub behavior, timestamp is treated as an offset. + * + * This consumer is designed to work with our KafkaProducer implementation. Our KafkaProducer adds attribute "offset" + * to the message, which is a timestamp measured just before sending the message (Producer's time). This offset is + * retrieved from Pub/Sub message and returned as offset for message. Offset attribute in message is necessary to + * process messages, lack of it causes an error. This timestamp is not necessarily equal to timestamp generated by + * Pub/Sub servers - this one is returned as "timestamp" attribute of ConsumerRecord. + * Note that due to this differences offset may repeat for more than one message and doing maths on it may not result + * in the same behavior as in Kafka. + * + * Value is kept in Pub/Sub message body. Key is kept in Pub/Sub "key" attribute. + * + * It uses code on extending deadlines and acknowledging messages based on Google Cloud Platform Pub/Sub Client Library + * (Subscriber). See this one for reference. + * + * Interceptors method onCommit has no mapping in Pub/Sub, so is not called. + * * @param Key * @param value */ @@ -148,6 +167,9 @@ private KafkaConsumer(Config configOptions) { ChannelUtil.getInstance().getCallCredentials()); } + /** + * All grpc stubs are reusing the same channel. + */ @SuppressWarnings("unchecked") KafkaConsumer(Config config, Channel channel, CallCredentials callCredentials) { try { @@ -155,8 +177,11 @@ private KafkaConsumer(Config configOptions) { Preconditions.checkNotNull(channel); - SubscriberFutureStub subscriberFutureStub = SubscriberGrpc.newFutureStub(channel); - PublisherFutureStub publisherFutureStub = PublisherGrpc.newFutureStub(channel); + SubscriberFutureStub subscriberFutureStub = SubscriberGrpc.newFutureStub(channel) + .withDeadlineAfter(config.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); + + PublisherFutureStub publisherFutureStub = PublisherGrpc.newFutureStub(channel) + .withDeadlineAfter(config.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); if(callCredentials != null) { subscriberFutureStub = subscriberFutureStub.withCallCredentials(callCredentials); @@ -174,6 +199,10 @@ private KafkaConsumer(Config configOptions) { } } + /** + * Assignment returns a set of the topics it is subscribed to. + * Set contains every topic once, combined with partition number 0 (default value). + */ @Override public Set assignment() { Set partitions = new HashSet<>(); @@ -183,14 +212,18 @@ public Set assignment() { return partitions; } + /** + * Returns set of topic names the consumer is subscribed to. + */ @Override public Set subscription() { return topicNameToSubscriber.keySet(); } /** - * Subscribe call tries to get existing subscriptions for groupId provided in configuration and topic names. If - * consumer was configured to create subscriptions if they don't exist, on failure with NOT_FOUND status it sends + * Subscribe call tries to get existing subscriptions for groupId provided in configuration and topic names. + * Subscription names are created with pattern topicName_groupId. + * If consumer was configured to create subscriptions if they don't exist, on failure with NOT_FOUND status it sends * "createSubscription" gRPC calls. If it was not configured to create subscriptions, throws KafkaException * if matching subscriptions do not exist. */ @@ -325,13 +358,17 @@ private void checkSubscribePreconditions(Collection topics) { } } + /** + * Subscribes to the collection of topics + * @param topics + */ @Override public void subscribe(Collection topics) { subscribe(topics, NO_REBALANCE_LISTENER); } /** - Subscribe is pulling all topics from PubSub project and subscribing to ones matching provided + This version of subscribe gets all topics from PubSub project and calls subscribe() for ones matching provided pattern. */ @Override @@ -373,6 +410,10 @@ private void checkPatternSubscribePreconditions(Pattern pattern) { "Topic pattern to subscribe to cannot be null"); } + /** + * Unsubscribe stops all subscriptions and threads extending deadlines. If configured, deletes subscriptions it + * was subscribed to. Resets all collections used to keep track of consumer's state. + */ @Override public void unsubscribe() { for(Subscriber s : topicNameToSubscriber.values()) { @@ -414,6 +455,24 @@ private void deleteSubscriptionsIfAllowed(Collection subscriptions Futures.addCallback(listListenableFuture, new DeleteSubscriptionCallback()); } + /** + * First, poll checks if any calls of seekToBeginning or seekToEnd vere invoked, and if true, seeks on specific + * topics. + * + * Poll works in Round Robin fashion. On each call, poll is performed on specific topic. If any messages were + * returned, it returns ConsumerRecords containing all polled data. If no messages were returned it retries poll + * on next topic, as long as it polls some messages or gets though all topics. + * + * When polling, deadline extensions are scheduled. Messages deadlines will be extended until commit call + * (manual config), until auto.commit.interval passes since previous commit (auto config) + * or until max.ack.extension.period passes (maximum on deadline extensions). + * + * If auto commit configured, every time poll is called it will check if auto.commit.interval passed since last + * commit. If it did, it will perform a commit. If auto commit used, it is crucial to make sure all previously polled + * messages were processed before call to another poll. + * + * This method will perform onConsume interceptor method, if any interceptors were configured. + */ public ConsumerRecords poll(long timeout) { checkPollPreconditions(timeout); @@ -523,6 +582,10 @@ private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, deserializedValue); } + /** + * This method performs the same way as subscribe. It get set of topics from provided collection and performs + * subscribe on them. Partitions are ignored. + */ @Override public void assign(Collection partitions) { Set topics = new HashSet<>(); @@ -532,27 +595,47 @@ public void assign(Collection partitions) { subscribe(topics); } + /** + * Commit previously polled messages in a synchronous way (blocking) + */ @Override public void commitSync() { commit(true); } + /** + * For every topic in this collection, all currently not commited (but polled) messages which offset is smaller or + * equal than one in OffsetAndMetadata object are being commited. This behavior is different with Kafka. Synchronous. + */ @Override public void commitSync(final Map offsets) { commitForTopicAndOffset(offsets, true); } + /** + * Commit previously polled messages in a asynchronous way (non-blocking) + */ @Override public void commitAsync() { commit(false); } + /** + * OffsetCommitCallback has no meaning in Pub/Sub, so it will not be invoked. + * This call works exactly the same as commitAsync() (no params) + */ @Override public void commitAsync(OffsetCommitCallback callback) { log.warn("OffsetCommitCallback is not supported and will not be invoked"); commitAsync(); } + /** + * For every topic in this collection, all currently not commited (but polled) messages which offset is smaller than + * one in OffsetAndMetadata object are being commited. This behavior is different with Kafka. Asynchronous. + * + * OffsetCommitCallback has no meaning in Pub/Sub, so this parameter is ignored. + */ @Override public void commitAsync(final Map offsets, OffsetCommitCallback callback) { @@ -584,16 +667,42 @@ private void commitForTopicAndOffset(Map offs } } + /** + * Call to seek performs Pub/Sub seek on topic provided in TopicPartition object. Partition is ignored. + * + * This call results in marking all messages before this offset as ACKed, and all messages after this + * timestamp as NACKed. THIS CALL CHANGES MESSAGES STATE ON THE SERVER. This is not consistent with Kafka API. + */ @Override public void seek(TopicPartition partition, long offset) { seekToTimestamp(Collections.singletonList(partition.topic()), offset); } + /** + * Call to seekToBeginning performs Pub/Sub seek on topic provided in TopicPartition object to the timestamp 0. + * Partition is ignored. + * + * This call results in marking all messages after offset 0 as NACKed. THIS CALL CHANGES MESSAGES STATE ON THE SERVER. + * This is not consistent with Kafka API. + * + * This call is evaluated in lazy way on nearest call to poll method. + */ @Override public void seekToBeginning(Collection partitions) { seekToPlace(partitions, Seek.BEGINNING); } + /** + * Call to seekToEnd performs Pub/Sub seek on topic provided in TopicPartition object to the current timestamp. + * Partition is ignored. + * + * This call results in marking all messages before current timestamp as ACKed. THIS CALL CHANGES MESSAGES STATE + * ON THE SERVER. + * This is not consistent with Kafka API. + * + * This call is evaluated in lazy way on nearest call to poll method (current timestamp will be timestamp of poll + * start). + */ @Override public void seekToEnd(Collection partitions) { seekToPlace(partitions, Seek.END); @@ -658,27 +767,42 @@ private void seekToTimestamp(Collection topics, long timestamp) { } } + /** + * This method has absolutely no meaning in Pub/Sub. Throws exception. + */ @Override public long position(TopicPartition partition) { throw new UnsupportedOperationException("This method has no mapping in PubSub"); } + /** + * This method has absolutely no meaning in Pub/Sub. Throws exception. + */ @Override public OffsetAndMetadata committed(TopicPartition partition) { throw new UnsupportedOperationException("This method has no mapping in PubSub "); } + /** + * Process metrics may be provided by Pub/Sub Stackdriver project. Returns empty map. + */ @Override public Map metrics() { return new HashMap<>(); } + /** + * Returns dummy partition with number 0 and dummy nodes. + */ @Override public List partitionsFor(String topic) { Node[] dummy = {new Node(0, "", 0)}; return Arrays.asList(new PartitionInfo(topic, DEFAULT_PARTITION, dummy[0], dummy, dummy)); } + /** + * Returns list of topics available on the server, paired with dummy partition (number 0, dummy nodes). + */ @Override public Map> listTopics() { Map> partitionTopicMap = new HashMap<>(); @@ -692,6 +816,9 @@ public Map> listTopics() { return partitionTopicMap; } + /** + * Blocks messages polling from all topics contained in this collection. + */ @Override public void pause(Collection partitions) { for(TopicPartition partition: partitions) { @@ -699,6 +826,9 @@ public void pause(Collection partitions) { } } + /** + * Resumes messages polling from all topics contained in this collection. + */ @Override public void resume(Collection partitions) { for(TopicPartition partition: partitions) { @@ -706,6 +836,9 @@ public void resume(Collection partitions) { } } + /** + * Returns set of topics for which polling is blocked, paired with dummy partition number equal to 0. + */ @Override public Set paused() { Set paused = new HashSet<>(); @@ -715,6 +848,9 @@ public Set paused() { return paused; } + /** + * As in our implementation timestamp is the offset, it just changes data format. + */ @Override public Map offsetsForTimes( Map timestampsToSearch) { @@ -727,11 +863,17 @@ public Map offsetsForTimes( return offsetsForTimes; } + /** + * For every topic it returns beginning offset - in our case 0 (smallest possible timestamp). + */ @Override public Map beginningOffsets(Collection partitions) { return getTopicPartitionOffsetMap(partitions, 0L); } + /** + * For every topic it returns end offset - in our case current timestamp (biggest possible timestamp until now). + */ @Override public Map endOffsets(Collection partitions) { long millis = new DateTime().getMillis(); @@ -747,9 +889,7 @@ private Map getTopicPartitionOffsetMap(Collection Date: Tue, 10 Oct 2017 12:20:50 -0700 Subject: [PATCH 138/140] Update mapped_api_develop from master (#131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixing POM file to get rid of maven warnings * Changed formatting checks to check for the Google formatting rules and adapted code to these rules. See https://google.github.io/styleguide/javaguide.html * Changed to older version of checkstyle to be compatible with JDK 1.7 * Revert "Changed to older version of checkstyle to be compatible with JDK 1.7" This reverts commit 7376b6a0a3e30bcd9266470e42ebd439076b7b1e. * Check Google code style only at the end and only using JDK1.8 * For the actual tests, stop at the integration-test phase rather than also verify. Run verify once, separately and using JDK8 only. * Reverting some changes in jms-light/pom.xml that might have somehow influenced the global behavior of maven * Added a timeout while waiting for the fake server to terminate * Fixing POM file to get rid of maven warnings * Changed formatting checks to check for the Google formatting rules and adapted code to these rules. See https://google.github.io/styleguide/javaguide.html * Changed to older version of checkstyle to be compatible with JDK 1.7 * Revert "Changed to older version of checkstyle to be compatible with JDK 1.7" This reverts commit 7376b6a0a3e30bcd9266470e42ebd439076b7b1e. * Check Google code style only at the end and only using JDK1.8 * For the actual tests, stop at the integration-test phase rather than also verify. Run verify once, separately and using JDK8 only. * Reverting some changes in jms-light/pom.xml that might have somehow influenced the global behavior of maven * Added a timeout while waiting for the fake server to terminate * Put tests back, but disable them * add go subscriber loadtest (#74) * typo * add go subscriber * automatically compile go loadtest binary * pr comment * Add PubSubTemporaryTopic class * Improves error messages, enables Go message tracking. * Update README.md * Update README.md * Update README.md * Fix null key and exception about read-only ByteBuffer that happens wh… (#78) * Fix null key and exception about read-only ByteBuffer that happens when using a JsonConverter in a source connector. * Fix null key and exception about read-only ByteBuffer that happens when using a JsonConverter in a source connector. * added source file * nit * nit * nit * use public access * use extends * Moved mockito and junit to the test scope. Removed the dependency on the kafka client. This is provided by the connect-api. Changed the scope of connect-api to provided. Updated the kafka version to 0.10.2.0. (#84) * nit * Initial version of PubSubMessageConsumer. * added file * added constructor * nit * fix pmd * nit * nit * added file * added -Xlint:all to all compile * Xlint Configuration This is the configuration that I used. With this, I get the following warnings (the second one is probably independent of -Xlint): ... [WARNING] /usr/local/google/home/uhu/src/pubsub/jms-light/src/main/java/com/google/pubsub/jms/light/message/PubSubTextMessage.java:[39,16] unchecked cast required: T found: java.lang.String ... [WARN] /usr/local/google/home/uhu/src/pubsub/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicSession.java:9: Wrong lexicographical order for 'com.google.pubsub.jms.light.destination.PubSubTemporaryTopic' import. Should be before 'javax.jms.TemporaryTopic'. [CustomImportOrder] [WARN] /usr/local/google/home/uhu/src/pubsub/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java:12: Line is longer than 100 characters (found 109). [LineLength] [WARN] /usr/local/google/home/uhu/src/pubsub/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java:20: Line is longer than 100 characters (found 114). [LineLength] * Resolved unchecked cast * Cleaned code style, especially for test classes. This addresses Github JMS issue #92. * nit * nit * Handle a null schema in the sink connector. (#94) * Adding ${os.detected.classifier} to Pom (#96) Just adding ${os.detected.classifier} to Pom so the right netty-tcnative-boring-ssl-static library is selected. This is to ensure the connector can work in any OS (e.g. OSX). * Fix concurrent access on iteration of synchronized set. (#98) * Handle a null schema in the sink connector. * Fix concurrent access on iteration of synchronized set. * Reduce the ackIds iteration synchronization to the shortest period needed * Make acks best effort. (#99) * Clear list of ackIds as soon as acks are sent. This will make acks best effort, but also prevents us from forgetting to send some acks if a poll happens while there is an outstanding ack request * Better error message when verifySubscription fails. (#100) * Better error message when verifySubscription fails. * When an exception happens on pull, just return an empty list to poll. (#102) * Handle a null schema in the sink connector. * Fix concurrent access on iteration of synchronized set. * Reduce the ackIds iteration synchronization to the shortest period needed * Clear list of ackIds as soon as acks are sent. This will make acks best effort, but also prevents us from forgetting to send some acks if a poll happens while there is an outstanding ack request * Better error message when verifySubscription fails. * When an exception happens on pull, just return an empty list to poll. * don't depend on gRPC internals (#103) * Don't reference gRPC internal classes * don't depend on gRPC internals * Create README.md * Ensure that partition numbers are non-negative (#105) * Ensure partition number is non-negative in CPS source connector. * Update client library dependencies. * Update Framework with Java Beta Client (#108) * Update Framework with Java Beta Client Updates the framework with the Java Beta client, and removes the experimental client now that the changes have been merged into the google-cloud-java repository. * Remove previous "spi" from import paths * Minor bugfixes, removed VTK since it no longer exists * Remove unused startup scripts (#110) * Remove Alpha Client Libraries, Redirect to google-cloud-java (#109) * Remove Alpha Client Libraries, Redirect to google-cloud-java * Removed client from .travis.yml * readme fix * lint fixes * Added Ruby publisher * updated publisher * Added Ruby subscriber * fixed comments * fixes for CPS Subscriber, updates with new Python publisher * python subscriber * fix firewall * Update run.py Fix for python3 * Shade the guava library to avoid a NoMethodFoundError (#117) * Handle a null schema in the sink connector. * Fix concurrent access on iteration of synchronized set. * Reduce the ackIds iteration synchronization to the shortest period needed * Clear list of ackIds as soon as acks are sent. This will make acks best effort, but also prevents us from forgetting to send some acks if a poll happens while there is an outstanding ack request * Better error message when verifySubscription fails. * When an exception happens on pull, just return an empty list to poll. * Ensure partition number is non-negative in CPS source connector. * Shade out the guava library so the connector can work with a newer version of Kafka that depends on a newer version of the guava library. * Shade out the guava library so the connector can work with a newer version of Kafka that depends on a newer version of the guava library. * fix * Fix some broken markdown in the README (#119) Some newlines snuck their way into these links * Update run.py Uncomment Go * java: don't sync on shutdown (#125) * java: don't sync on shutdown Shutting down needs to wait for messages to be processed, and processing the messages need to lock the Task object. So, if the thread calling shutdown locks the Task object, we'll deadlock. * pr comment * increase parallel puller count (#124) * increase parallel puller count The new default number of parallel pullers is NumCPU. To keep up with publishers, we need to increase the number of pullers. This commit increaes the number to 5*NumCPU, the same as the old default. * bump version * Move away from very old versions of gRPC libraries (#130) * Handle a null schema in the sink connector. * Fix concurrent access on iteration of synchronized set. * Reduce the ackIds iteration synchronization to the shortest period needed * Clear list of ackIds as soon as acks are sent. This will make acks best effort, but also prevents us from forgetting to send some acks if a poll happens while there is an outstanding ack request * Better error message when verifySubscription fails. * When an exception happens on pull, just return an empty list to poll. * Ensure partition number is non-negative in CPS source connector. * Shade out the guava library so the connector can work with a newer version of Kafka that depends on a newer version of the guava library. * Shade out the guava library so the connector can work with a newer version of Kafka that depends on a newer version of the guava library. * Update versions of gRPC libraries used by Kafka Connector. This should hopefully fix issue #120. * Edit travis script. --- .travis.yml | 5 +- client-samples/README.md | 16 - client-samples/pom.xml | 58 -- .../google/cloud/pubsub/PublisherSamples.java | 61 --- .../cloud/pubsub/SubscriberSamples.java | 55 -- client/README.md | 18 +- client/pom.xml | 221 -------- .../pubsub/AbstractSubscriberConnection.java | 491 ----------------- .../com/google/cloud/pubsub/Distribution.java | 157 ------ .../google/cloud/pubsub/FlowController.java | 83 --- .../google/cloud/pubsub/MessagesWaiter.java | 67 --- .../pubsub/PollingSubscriberConnection.java | 196 ------- .../com/google/cloud/pubsub/Publisher.java | 364 ------------- .../google/cloud/pubsub/PublisherImpl.java | 452 ---------------- .../google/cloud/pubsub/PublisherStats.java | 59 --- .../pubsub/StreamingSubscriberConnection.java | 216 -------- .../com/google/cloud/pubsub/Subscriber.java | 266 ---------- .../google/cloud/pubsub/SubscriberImpl.java | 321 ----------- .../google/cloud/pubsub/SubscriberStats.java | 80 --- .../google/cloud/pubsub/FakeCredentials.java | 58 -- .../pubsub/FakePublisherServiceImpl.java | 118 ----- .../pubsub/FakeScheduledExecutorService.java | 299 ----------- .../pubsub/FakeSubscriberServiceImpl.java | 400 -------------- .../cloud/pubsub/FlowControllerTest.java | 189 ------- .../cloud/pubsub/MessagesWaiterTest.java | 55 -- .../cloud/pubsub/PublisherImplTest.java | 462 ---------------- .../cloud/pubsub/SubscriberImplTest.java | 498 ------------------ client/src/test/resources/log4j.xml | 14 - client/src/test/resources/logging.properties | 5 - .../configs/checkstyle/google_checks.xml | 218 ++++++++ jms-light/configs/checkstyle/rules.xml | 193 ------- jms-light/pom.xml | 21 +- .../jms/light/AbstractMessageProducer.java | 75 +-- .../pubsub/jms/light/PubSubConnection.java | 122 ++--- .../jms/light/PubSubConnectionFactory.java | 45 +- .../jms/light/PubSubMessageProducer.java | 83 ++- .../pubsub/jms/light/PubSubSession.java | 32 +- .../jms/light/PubSubTopicConnection.java | 60 +++ .../light/PubSubTopicConnectionFactory.java | 28 + .../jms/light/PubSubTopicPublisher.java | 39 +- .../pubsub/jms/light/PubSubTopicSession.java | 61 +++ .../jms/light/consumer/PubSubMessageConsumer | 147 ++++++ .../light/destination/PubSubDestination.java | 5 +- .../destination/PubSubTemporaryTopic.java | 33 ++ .../destination/PubSubTopicDestination.java | 9 +- .../message/AbstractPropertyMessage.java | 70 +-- .../light/message/AbstractPubSubMessage.java | 87 +-- .../jms/light/message/PubSubTextMessage.java | 31 +- .../jms/light/session/AbstractSession.java | 30 +- .../AbstractSessionBrowserCreator.java | 22 +- .../AbstractSessionConsumerCreator.java | 68 +-- .../AbstractSessionMessageCreator.java | 40 +- .../AbstractSessionProducerCreator.java | 27 +- .../session/AbstractSessionQueueCreator.java | 20 +- .../AbstractSessionSubscriberCreator.java | 30 +- .../session/AbstractSessionTopicCreator.java | 20 +- .../integration/BaseIntegrationTest.java | 27 +- .../integration/PublisherIntegrationTest.java | 80 ++- kafka-connector/README.md | 17 +- kafka-connector/pom.xml | 25 +- .../kafka/sink/CloudPubSubSinkTask.java | 5 +- .../source/CloudPubSubSourceConnector.java | 2 +- .../kafka/source/CloudPubSubSourceTask.java | 36 +- .../kafka/sink/CloudPubSubSinkTaskTest.java | 10 + .../source/CloudPubSubSourceTaskTest.java | 78 ++- load-test-framework/README.md | 8 +- load-test-framework/pom.xml | 53 +- .../python_src/clients/cps_publisher_task.py | 38 +- .../python_src/clients/cps_subscriber_task.py | 74 +++ .../python_src/clients/loadtest_pb2.py | 154 ++++-- .../python_src/clients/requirements.txt | 1 - load-test-framework/ruby_src/Gemfile | 3 + load-test-framework/ruby_src/Gemfile.lock | 75 +++ .../ruby_src/cps_publisher_task.rb | 62 +++ .../ruby_src/cps_subscriber_task.rb | 53 ++ load-test-framework/ruby_src/loadtest_pb.rb | 76 +++ .../ruby_src/loadtest_services_pb.rb | 48 ++ load-test-framework/run.py | 44 +- .../experimental/CPSPublisherTask.java | 102 ---- .../experimental/CPSSubscriberTask.java | 91 ---- .../clients/gcloud/CPSPublisherTask.java | 92 ++-- .../clients/gcloud/CPSSubscriberTask.java | 40 +- .../pubsub/clients/vtk/CPSPublisherTask.java | 112 ---- .../java/com/google/pubsub/flic/Driver.java | 110 ++-- .../pubsub/flic/controllers/Client.java | 69 ++- .../flic/controllers/GCEController.java | 6 +- .../pubsub/flic/output/SheetsService.java | 29 +- ...erimental-java-publisher_startup_script.sh | 31 -- .../cps-gcloud-go-publisher_startup_script.sh | 11 +- ...ps-gcloud-go-subscriber_startup_script.sh} | 16 +- ...-gcloud-python-publisher_startup_script.sh | 12 +- ...cloud-python-subscriber_startup_script.sh} | 24 +- ...ps-gcloud-ruby-publisher_startup_script.sh | 48 ++ ...s-gcloud-ruby-subscriber_startup_script.sh | 48 ++ 94 files changed, 1964 insertions(+), 6746 deletions(-) delete mode 100644 client-samples/README.md delete mode 100755 client-samples/pom.xml delete mode 100644 client-samples/src/main/java/com/google/cloud/pubsub/PublisherSamples.java delete mode 100644 client-samples/src/main/java/com/google/cloud/pubsub/SubscriberSamples.java delete mode 100755 client/pom.xml delete mode 100644 client/src/main/java/com/google/cloud/pubsub/AbstractSubscriberConnection.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/Distribution.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/FlowController.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/MessagesWaiter.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/PollingSubscriberConnection.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/Publisher.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/PublisherImpl.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/PublisherStats.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/StreamingSubscriberConnection.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/Subscriber.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/SubscriberImpl.java delete mode 100644 client/src/main/java/com/google/cloud/pubsub/SubscriberStats.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/FakeCredentials.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/FakePublisherServiceImpl.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/FakeScheduledExecutorService.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/FakeSubscriberServiceImpl.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/FlowControllerTest.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/MessagesWaiterTest.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/PublisherImplTest.java delete mode 100644 client/src/test/java/com/google/cloud/pubsub/SubscriberImplTest.java delete mode 100644 client/src/test/resources/log4j.xml delete mode 100644 client/src/test/resources/logging.properties create mode 100644 jms-light/configs/checkstyle/google_checks.xml delete mode 100644 jms-light/configs/checkstyle/rules.xml create mode 100644 jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnection.java create mode 100644 jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java create mode 100644 jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicSession.java create mode 100644 jms-light/src/main/java/com/google/pubsub/jms/light/consumer/PubSubMessageConsumer create mode 100644 jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTemporaryTopic.java create mode 100644 load-test-framework/python_src/clients/cps_subscriber_task.py create mode 100644 load-test-framework/ruby_src/Gemfile create mode 100644 load-test-framework/ruby_src/Gemfile.lock create mode 100644 load-test-framework/ruby_src/cps_publisher_task.rb create mode 100644 load-test-framework/ruby_src/cps_subscriber_task.rb create mode 100644 load-test-framework/ruby_src/loadtest_pb.rb create mode 100644 load-test-framework/ruby_src/loadtest_services_pb.rb delete mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSPublisherTask.java delete mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSSubscriberTask.java delete mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/vtk/CPSPublisherTask.java delete mode 100644 load-test-framework/src/main/resources/gce/cps-experimental-java-publisher_startup_script.sh rename load-test-framework/src/main/resources/gce/{cps-experimental-java-subscriber_startup_script.sh => cps-gcloud-go-subscriber_startup_script.sh} (65%) rename load-test-framework/src/main/resources/gce/{cps-vtk-java-publisher_startup_script.sh => cps-gcloud-python-subscriber_startup_script.sh} (50%) create mode 100755 load-test-framework/src/main/resources/gce/cps-gcloud-ruby-publisher_startup_script.sh create mode 100755 load-test-framework/src/main/resources/gce/cps-gcloud-ruby-subscriber_startup_script.sh diff --git a/.travis.yml b/.travis.yml index 139eda3a..f52a4979 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,9 @@ matrix: jdk: openjdk7 script: - - mvn -q -B -f jms-light/pom.xml clean verify + - mvn -q -B -f jms-light/pom.xml clean integration-test - mvn -q -B -f kafka-connector/pom.xml clean verify - - mvn -q -B -f client/pom.xml clean verify -Dmaven.javadoc.skip=true -Dgpg.skip=true -Dmaven.test.skip=true -Djava.util.logging.config.file=client/src/test/resources/logging.properties - jdk_switcher use oraclejdk8 - mvn -q -B -f pubsub-mapped-api/pom.xml clean install verify - mvn -q -B -f load-test-framework/pom.xml clean install verify - + - mvn -q -B -f jms-light/pom.xml clean verify -Dmaven.test.skip=true \ No newline at end of file diff --git a/client-samples/README.md b/client-samples/README.md deleted file mode 100644 index e05ea19c..00000000 --- a/client-samples/README.md +++ /dev/null @@ -1,16 +0,0 @@ -### Introduction - -Code samples for google3/third_party/java_src/cloud/pubsub/project/client. - -(NOTE) This client includes only data plane operations, publishing and -pulling messages, and it is based on our gRPC beta release. - -(ALPHA) This library uses features that are part of an invitation-only -release of the underlying Cloud Pub/Sub API. The library will generate -errors unless you have access to this API. This restriction should be -relaxed in the near future. Please contact cloud-pubsub@google.com with any -questions in the meantime. - -### How to use - -blaze run third_party/java_src/cloud/pubsub/project:client_[publisher|subscriber]_samples -- [your topic|subscription] diff --git a/client-samples/pom.xml b/client-samples/pom.xml deleted file mode 100755 index 84279b69..00000000 --- a/client-samples/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - 4.0.0 - - com.google.pubsub - cloud-pubsub-client-samples - jar - 0.2-EXPERIMENTAL - - Google Cloud Pub/Sub Client Code Samples - - Code samples for the High performance Cloud Pub/Sub client library. - - (ALPHA) This samples use features that are part of an invitation-only - release of the underlying Cloud Pub/Sub API. The library will generate - errors unless you have access to this API. This restriction should be - relaxed in the near future. Please contact cloud-pubsub@google.com with any - questions in the meantime. - - https://cloud.google.com/pubsub - - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - Google Inc. - http://www.google.com - - - - - - com.google.pubsub - cloud-pubsub-client - 0.1-EXPERIMENTAL - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.6.0 - - 1.8 - 1.8 - - - - - diff --git a/client-samples/src/main/java/com/google/cloud/pubsub/PublisherSamples.java b/client-samples/src/main/java/com/google/cloud/pubsub/PublisherSamples.java deleted file mode 100644 index b884c1cb..00000000 --- a/client-samples/src/main/java/com/google/cloud/pubsub/PublisherSamples.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PubsubMessage; - -/** Some sensible samples that demonstrate how to use the {@link Publisher}. */ -public class PublisherSamples { - - private PublisherSamples() {} - - @SuppressWarnings("unchecked") - public static void publish(String topic, String message) throws Exception { - Publisher publisher = Publisher.Builder.newBuilder(topic).build(); - - Futures.addCallback( - publisher.publish( - PubsubMessage.newBuilder() - .setData(ByteString.copyFromUtf8(message)) - .build()), - new FutureCallback() { - @Override - public void onSuccess(String messageId) { - System.out.println("Message " + messageId + " published successfully."); - } - - @Override - public void onFailure(Throwable thrown) { - System.out.println("Failed to publish message: " + thrown); - } - }); - publisher.shutdown(); - } - - public static void main(String[] args) throws Exception { - if (args.length != 2) { - System.out.println("You must specify a topic and the message to publish as parameters."); - System.exit(1); - return; - } - - PublisherSamples.publish(args[0], args[1]); - } -} diff --git a/client-samples/src/main/java/com/google/cloud/pubsub/SubscriberSamples.java b/client-samples/src/main/java/com/google/cloud/pubsub/SubscriberSamples.java deleted file mode 100644 index 16bbd7fd..00000000 --- a/client-samples/src/main/java/com/google/cloud/pubsub/SubscriberSamples.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.cloud.pubsub.Subscriber.MessageReceiver; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.pubsub.v1.PubsubMessage; - -/** Some sensible samples that demonstrate how to use the {@link Subscriber}. */ -public class SubscriberSamples { - - private SubscriberSamples() {} - - @SuppressWarnings("unchecked") - public static void subscribe(String subscription) throws Exception { - Subscriber.Builder.newBuilder( - subscription, - new MessageReceiver() { - @Override - public ListenableFuture receiveMessage(PubsubMessage message) { - System.out.println(message.getData()); - return Futures.immediateFuture(AckReply.ACK); - } - }) - .build() - .startAsync() - .awaitTerminated(); - } - - public static void main(String[] args) throws Exception { - if (args.length == 0) { - System.out.println("You must specify a subscription path of the form " - + "projects/{project}/subscriptions/{subscription} as a parameter."); - System.exit(1); - return; - } - - SubscriberSamples.subscribe(args[0]); - } -} diff --git a/client/README.md b/client/README.md index 6e8b7e36..d733187b 100644 --- a/client/README.md +++ b/client/README.md @@ -1,12 +1,6 @@ -### Introduction - -High Performance Cloud Pub/Sub Client libraries. - -(NOTE) This client includes only data plane operations, publishing and -pulling messages, and it is based on our gRPC beta release. - -(ALPHA) This library uses features that are part of an invitation-only -release of the underlying Cloud Pub/Sub API. The library will generate -errors unless you have access to this API. This restriction should be -relaxed in the near future. Please contact cloud-pubsub@google.com with any -questions in the meantime. +The alpha client libraries in this directory have been replaced by the official +[Google Cloud Pub/Sub Java client library](https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub). +The semantics and performance improvements introduced in these alpha libraries +have been preserved and improved in the official libraries, so we strongly +encourage you to migrate. However, we have no plan to remove this code from our [Maven +repository](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.google.pubsub%22). diff --git a/client/pom.xml b/client/pom.xml deleted file mode 100755 index 65ca04dc..00000000 --- a/client/pom.xml +++ /dev/null @@ -1,221 +0,0 @@ - - 4.0.0 - - com.google.pubsub - cloud-pubsub-client - jar - 0.2-EXPERIMENTAL - - Google Cloud Pub/Sub Client - - High performance Cloud Pub/Sub client library. - - (NOTE) This release includes only data plane operations, publishing and - pulling messages, and it is based on our gRPC beta release. - - (ALPHA) This library uses features that are part of an invitation-only - release of the underlying Cloud Pub/Sub API. The library will generate - errors unless you have access to this API. This restriction should be - relaxed in the near future. Please contact cloud-pubsub@google.com with any - questions in the meantime. - - https://cloud.google.com/pubsub - - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - Google Inc. - http://www.google.com - - - - - scm:git:git@github.com:GoogleCloudPlatform/pubsub.git - scm:git:git@github.com:GoogleCloudPlatform/pubsub.git - https://github.com/GoogleCloudPlatform/pubsub/tree/master/client - - - - 1.0.1 - UTF-8 - - - - - com.google.auth - google-auth-library-oauth2-http - 0.5.0 - - - com.google.guava - guava - 20.0 - - - com.google.api.grpc - grpc-google-cloud-pubsub-v1 - 0.1.5 - - - io.grpc - grpc-netty - ${grpc.version} - - - io.grpc - grpc-auth - ${grpc.version} - - - io.grpc - grpc-protobuf - ${grpc.version} - - - io.grpc - grpc-stub - ${grpc.version} - - - io.netty - netty-tcnative-boringssl-static - 1.1.33.Fork23 - - - joda-time - joda-time - 2.9.4 - - - org.slf4j - slf4j-api - 1.7.21 - - - org.slf4j - slf4j-log4j12 - 1.7.21 - - - log4j - log4j - 1.2.17 - - - junit - junit - 4.11 - test - - - org.mockito - mockito-all - 1.9.5 - test - - - - - - - kr.motd.maven - os-maven-plugin - 1.4.0.Final - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - - cloud-pubsub-client-experimental-with-dependencies - package - - shade - - - false - cloud-pubsub-client-experimental-with-dependencies - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.6.0 - - 1.7 - 1.7 - - - - - org.apache.maven.plugins - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.3 - true - - ossrh - https://oss.sonatype.org/ - true - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - -Xdoclint:none - - - - - - diff --git a/client/src/main/java/com/google/cloud/pubsub/AbstractSubscriberConnection.java b/client/src/main/java/com/google/cloud/pubsub/AbstractSubscriberConnection.java deleted file mode 100644 index 236fe902..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/AbstractSubscriberConnection.java +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.cloud.pubsub.Publisher.CloudPubsubFlowControlException; -import com.google.cloud.pubsub.Subscriber.MessageReceiver; -import com.google.cloud.pubsub.Subscriber.MessageReceiver.AckReply; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import com.google.common.primitives.Ints; -import com.google.common.util.concurrent.AbstractService; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.ReceivedMessage; -import io.grpc.Status; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.joda.time.Interval; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Abstract base implementation class of a subscriber connection in charge of receiving subscription - * messages. - */ -abstract class AbstractSubscriberConnection extends AbstractService { - private static final Logger logger = LoggerFactory.getLogger(AbstractSubscriberConnection.class); - - private static final int INITIAL_ACK_DEADLINE_EXTENSION_SECONDS = 2; - @VisibleForTesting static final Duration PENDING_ACKS_SEND_DELAY = Duration.millis(100); - private static final int MAX_ACK_DEADLINE_EXTENSION_SECS = 10 * 60; // 10m - - protected final String subscription; - protected final ScheduledExecutorService executor; - - private final Duration ackExpirationPadding; - private final MessageReceiver receiver; - - private final FlowController flowController; - private final MessagesWaiter messagesWaiter; - - // Map of outstanding messages (value) ordered by expiration time (key) in ascending order. - private final Map> outstandingAckHandlers; - private final Set pendingAcks; - private final Set pendingNacks; - - private final Lock alarmsLock; - private int messageDeadlineSeconds; - private ScheduledFuture ackDeadlineExtensionAlarm; - private Instant nextAckDeadlineExtensionAlarmTime; - private ScheduledFuture pendingAcksAlarm; - - // To keep track of number of seconds the receiver takes to process messages. - private final Distribution ackLatencyDistribution; - - private static class ExpirationInfo implements Comparable { - Instant expiration; - int nextExtensionSeconds; - - ExpirationInfo(Instant expiration, int initialAckDeadlineExtension) { - this.expiration = expiration; - nextExtensionSeconds = initialAckDeadlineExtension; - } - - void extendExpiration() { - expiration = Instant.now().plus(Duration.standardSeconds(nextExtensionSeconds)); - nextExtensionSeconds = 2 * nextExtensionSeconds; - if (nextExtensionSeconds > MAX_ACK_DEADLINE_EXTENSION_SECS) { - nextExtensionSeconds = MAX_ACK_DEADLINE_EXTENSION_SECS; - } - } - - @Override - public int hashCode() { - return expiration.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ExpirationInfo)) { - return false; - } - - ExpirationInfo other = (ExpirationInfo) obj; - return expiration.equals(other.expiration); - } - - @Override - public int compareTo(ExpirationInfo other) { - return expiration.compareTo(other.expiration); - } - } - - /** Stores the data needed to asynchronously modify acknowledgement deadlines. */ - static class PendingModifyAckDeadline { - final List ackIds; - final int deadlineExtensionSeconds; - - PendingModifyAckDeadline(int deadlineExtensionSeconds) { - this.ackIds = new ArrayList(); - this.deadlineExtensionSeconds = deadlineExtensionSeconds; - } - - PendingModifyAckDeadline(String ackId, int deadlineExtensionSeconds) { - this(deadlineExtensionSeconds); - addAckId(ackId); - } - - public void addAckId(String ackId) { - ackIds.add(ackId); - } - } - - /** Handles callbacks for acking/nacking messages from the {@link MessageReceiver}. */ - private class AckHandler implements FutureCallback { - private final String ackId; - private final int outstandingBytes; - private final AtomicBoolean acked; - private final Instant receivedTime; - - AckHandler(String ackId, int outstandingBytes) { - this.ackId = ackId; - this.outstandingBytes = outstandingBytes; - acked = new AtomicBoolean(false); - receivedTime = Instant.now(); - } - - @Override - public void onFailure(Throwable t) { - logger.warn( - "MessageReceiver failed to processes ack ID: " + ackId + ", the message will be nacked.", - t); - synchronized (pendingNacks) { - pendingNacks.add(ackId); - } - setupPendingAcksAlarm(); - flowController.release(1, outstandingBytes); - messagesWaiter.incrementPendingMessages(-1); - } - - @Override - public void onSuccess(AckReply reply) { - acked.getAndSet(true); - switch (reply) { - case ACK: - synchronized (pendingAcks) { - pendingAcks.add(ackId); - } - setupPendingAcksAlarm(); - flowController.release(1, outstandingBytes); - // Record the latency rounded to the next closest integer. - ackLatencyDistribution.record( - Ints.saturatedCast( - (long) Math.ceil(new Duration(receivedTime, Instant.now()).getMillis() / 1000D))); - messagesWaiter.incrementPendingMessages(-1); - return; - case NACK: - synchronized (pendingNacks) { - pendingNacks.add(ackId); - } - setupPendingAcksAlarm(); - flowController.release(1, outstandingBytes); - messagesWaiter.incrementPendingMessages(-1); - return; - default: - throw new IllegalArgumentException(String.format("AckReply: %s not supported", reply)); - } - } - } - - AbstractSubscriberConnection( - String subscription, - MessageReceiver receiver, - Duration ackExpirationPadding, - Distribution ackLatencyDistribution, - FlowController flowController, - ScheduledExecutorService executor) { - this.executor = executor; - this.ackExpirationPadding = ackExpirationPadding; - this.receiver = receiver; - this.subscription = subscription; - this.flowController = flowController; - outstandingAckHandlers = new HashMap<>(); - pendingAcks = new HashSet<>(); - pendingNacks = new HashSet<>(); - // 601 buckets of 1s resolution from 0s to MAX_ACK_DEADLINE_SECONDS - this.ackLatencyDistribution = ackLatencyDistribution; - alarmsLock = new ReentrantLock(); - nextAckDeadlineExtensionAlarmTime = new Instant(Long.MAX_VALUE); - messagesWaiter = new MessagesWaiter(); - } - - @Override - protected void doStart() { - logger.debug("Starting subscriber."); - initialize(); - notifyStarted(); - } - - abstract void initialize(); - - @Override - protected void doStop() { - messagesWaiter.waitNoMessages(); - alarmsLock.lock(); - try { - if (ackDeadlineExtensionAlarm != null) { - ackDeadlineExtensionAlarm.cancel(true); - ackDeadlineExtensionAlarm = null; - } - } finally { - alarmsLock.unlock(); - } - processOutstandingAckOperations(); - notifyStopped(); - } - - protected boolean isRetryable(Status status) { - switch (status.getCode()) { - case DEADLINE_EXCEEDED: - case INTERNAL: - case CANCELLED: - case RESOURCE_EXHAUSTED: - case UNAVAILABLE: - return true; - default: - return false; - } - } - - protected void setMessageDeadlineSeconds(int messageDeadlineSeconds) { - this.messageDeadlineSeconds = messageDeadlineSeconds; - } - - protected int getMessageDeadlineSeconds() { - return messageDeadlineSeconds; - } - - protected void processReceivedMessages( - List responseMessages) { - int receivedMessagesCount = responseMessages.size(); - if (receivedMessagesCount == 0) { - return; - } - Instant now = Instant.now(); - int totalByteCount = 0; - final List ackHandlers = new ArrayList<>(responseMessages.size()); - for (ReceivedMessage pubsubMessage : responseMessages) { - int messageSize = pubsubMessage.getMessage().getSerializedSize(); - totalByteCount += messageSize; - ackHandlers.add(new AckHandler(pubsubMessage.getAckId(), messageSize)); - } - ExpirationInfo expiration = - new ExpirationInfo( - now.plus(messageDeadlineSeconds * 1000), INITIAL_ACK_DEADLINE_EXTENSION_SECONDS); - synchronized (outstandingAckHandlers) { - addOutstadingAckHandlers(expiration, ackHandlers); - } - logger.debug("Received {} messages at {}", responseMessages.size(), now); - setupNextAckDeadlineExtensionAlarm(expiration); - - messagesWaiter.incrementPendingMessages(responseMessages.size()); - Iterator acksIterator = ackHandlers.iterator(); - for (ReceivedMessage userMessage : responseMessages) { - final PubsubMessage message = userMessage.getMessage(); - final AckHandler ackHandler = acksIterator.next(); - executor.submit( - new Runnable() { - @Override - public void run() { - Futures.addCallback(receiver.receiveMessage(message), ackHandler); - } - }); - } - try { - flowController.reserve(receivedMessagesCount, totalByteCount); - } catch (CloudPubsubFlowControlException unexpectedException) { - throw new IllegalStateException("Flow control unexpected exception", unexpectedException); - } - } - - private void addOutstadingAckHandlers( - ExpirationInfo expiration, final List ackHandlers) { - if (outstandingAckHandlers.get(expiration) == null) - { - outstandingAckHandlers.put(expiration, new ArrayList(ackHandlers.size())); - } - outstandingAckHandlers.get(expiration).addAll(ackHandlers); - } - - private void setupPendingAcksAlarm() { - alarmsLock.lock(); - try { - if (pendingAcksAlarm == null) { - pendingAcksAlarm = - executor.schedule( - new Runnable() { - @Override - public void run() { - alarmsLock.lock(); - try { - pendingAcksAlarm = null; - } finally { - alarmsLock.unlock(); - } - processOutstandingAckOperations(); - } - }, - PENDING_ACKS_SEND_DELAY.getMillis(), - TimeUnit.MILLISECONDS); - } - } finally { - alarmsLock.unlock(); - } - } - - private class AckDeadlineAlarm implements Runnable { - @Override - public void run() { - alarmsLock.lock(); - try { - nextAckDeadlineExtensionAlarmTime = new Instant(Long.MAX_VALUE); - ackDeadlineExtensionAlarm = null; - if (pendingAcksAlarm != null) { - pendingAcksAlarm.cancel(false); - pendingAcksAlarm = null; - } - } finally { - alarmsLock.unlock(); - } - - Instant now = Instant.now(); - // Rounded to the next second, so we only schedule future alarms at the second - // resolution. - Instant cutOverTime = - new Instant( - ((long) Math.ceil(now.plus(ackExpirationPadding).plus(500).getMillis() / 1000.0)) - * 1000L); - logger.debug( - "Running alarm sent outstanding acks, at now time: {}, with cutover " - + "time: {}, padding: {}", - now, - cutOverTime, - ackExpirationPadding); - ExpirationInfo nextScheduleExpiration = null; - List modifyAckDeadlinesToSend = new ArrayList<>(); - - synchronized (outstandingAckHandlers) { - for (ExpirationInfo messageExpiration : outstandingAckHandlers.keySet()) { - if (messageExpiration.expiration.compareTo(cutOverTime) <= 0) { - Collection expiringAcks = outstandingAckHandlers.get(messageExpiration); - outstandingAckHandlers.remove(messageExpiration); - List renewedAckHandlers = new ArrayList<>(expiringAcks.size()); - messageExpiration.extendExpiration(); - int extensionSeconds = - Ints.saturatedCast( - new Interval(now, messageExpiration.expiration) - .toDuration() - .getStandardSeconds()); - PendingModifyAckDeadline pendingModAckDeadline = - new PendingModifyAckDeadline(extensionSeconds); - for (AckHandler ackHandler : expiringAcks) { - if (ackHandler.acked.get()) { - continue; - } - pendingModAckDeadline.addAckId(ackHandler.ackId); - renewedAckHandlers.add(ackHandler); - } - modifyAckDeadlinesToSend.add(pendingModAckDeadline); - if (!renewedAckHandlers.isEmpty()) { - addOutstadingAckHandlers(messageExpiration, renewedAckHandlers); - } else { - outstandingAckHandlers.remove(messageExpiration); - } - } - if (nextScheduleExpiration == null - || nextScheduleExpiration.expiration.isAfter(messageExpiration.expiration)) { - nextScheduleExpiration = messageExpiration; - } - } - } - - processOutstandingAckOperations(modifyAckDeadlinesToSend); - - if (nextScheduleExpiration != null) { - logger.debug( - "Scheduling based on outstanding, now time: {}, " + "next schedule time: {}", - now, - nextScheduleExpiration); - setupNextAckDeadlineExtensionAlarm(nextScheduleExpiration); - } - } - } - - private void setupNextAckDeadlineExtensionAlarm(ExpirationInfo messageExpiration) { - Instant possibleNextAlarmTime = messageExpiration.expiration.minus(ackExpirationPadding); - alarmsLock.lock(); - try { - if (nextAckDeadlineExtensionAlarmTime.isAfter(possibleNextAlarmTime)) { - logger.debug( - "Scheduling next alarm time: {}, last alarm set to time: {}", - possibleNextAlarmTime, - nextAckDeadlineExtensionAlarmTime); - if (ackDeadlineExtensionAlarm != null) { - logger.debug("Canceling previous alarm"); - ackDeadlineExtensionAlarm.cancel(false); - } - - nextAckDeadlineExtensionAlarmTime = possibleNextAlarmTime; - - ackDeadlineExtensionAlarm = - executor.schedule( - new AckDeadlineAlarm(), - nextAckDeadlineExtensionAlarmTime.getMillis() - Instant.now().getMillis(), - TimeUnit.MILLISECONDS); - } - - } finally { - alarmsLock.unlock(); - } - } - - private void processOutstandingAckOperations() { - processOutstandingAckOperations(new ArrayList()); - } - - private void processOutstandingAckOperations( - List ackDeadlineExtensions) { - List modifyAckDeadlinesToSend = - Lists.newArrayList(ackDeadlineExtensions); - List acksToSend = new ArrayList<>(pendingAcks.size()); - synchronized (pendingAcks) { - if (!pendingAcks.isEmpty()) { - try { - acksToSend = new ArrayList<>(pendingAcks); - logger.debug("Sending {} acks", acksToSend.size()); - } finally { - pendingAcks.clear(); - } - } - } - PendingModifyAckDeadline nacksToSend = new PendingModifyAckDeadline(0); - synchronized (pendingNacks) { - if (!pendingNacks.isEmpty()) { - try { - for (String ackId : pendingNacks) { - nacksToSend.addAckId(ackId); - } - logger.debug("Sending {} nacks", pendingNacks.size()); - } finally { - pendingNacks.clear(); - } - modifyAckDeadlinesToSend.add(nacksToSend); - } - } - - sendAckOperations(acksToSend, modifyAckDeadlinesToSend); - } - - abstract void sendAckOperations( - List acksToSend, List ackDeadlineExtensions); -} diff --git a/client/src/main/java/com/google/cloud/pubsub/Distribution.java b/client/src/main/java/com/google/cloud/pubsub/Distribution.java deleted file mode 100644 index a4f4878d..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/Distribution.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.base.MoreObjects; -import com.google.common.base.MoreObjects.ToStringHelper; -import com.google.common.base.Preconditions; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Takes measurements and stores them in linear buckets from 0 to totalBuckets - 1, along with - * utilities to calculate percentiles for analysis of results. - */ -public class Distribution { - - private final AtomicLong[] bucketCounts; - private long count; - private double mean; - private double sumOfSquaredDeviation; - - public Distribution(int totalBuckets) { - Preconditions.checkArgument(totalBuckets > 0); - bucketCounts = new AtomicLong[totalBuckets]; - for (int i = 0; i < totalBuckets; ++i) { - bucketCounts[i] = new AtomicLong(); - } - } - - /** - * Get the bucket that records values up to the given percentile. - */ - public long getNthPercentile(double percentile) { - Preconditions.checkArgument(percentile > 0.0); - Preconditions.checkArgument(percentile <= 100.0); - - long[] bucketCounts = getBucketCounts(); - long total = 0; - for (long count : bucketCounts) { - total += count; - } - - if (total == 0) { - return 0; - } - long count = (long) Math.ceil(total * percentile / 100.0); - for (int i = 0; i < bucketCounts.length; i++) { - count -= bucketCounts[i]; - if (count <= 0) { - return i; - } - } - return 0; - } - - /** - * Resets (sets to 0) the recorded values. - */ - public synchronized void reset() { - for (AtomicLong element : bucketCounts) { - element.set(0); - } - count = 0; - mean = 0; - sumOfSquaredDeviation = 0; - } - - /** - * Numbers of values recorded. - */ - public long getCount() { - return count; - } - - /** - * Square deviations of the recorded values. - */ - public double getSumOfSquareDeviations() { - return sumOfSquaredDeviation; - } - - /** - * Mean of the recorded values. - */ - public double getMean() { - return mean; - } - - /** - * Gets the accumulated count of every bucket of the distribution. - */ - public long[] getBucketCounts() { - long[] counts = new long[bucketCounts.length]; - for (int i = 0; i < counts.length; i++) { - counts[i] = bucketCounts[i].longValue(); - } - return counts; - } - - /** - * Make a copy of the distribution. - */ - public synchronized Distribution copy() { - Distribution distributionCopy = new Distribution(bucketCounts.length); - distributionCopy.count = count; - distributionCopy.mean = mean; - distributionCopy.sumOfSquaredDeviation = sumOfSquaredDeviation; - System.arraycopy(bucketCounts, 0, distributionCopy.bucketCounts, 0, bucketCounts.length); - return distributionCopy; - } - - /** - * Record a new value. - */ - public void record(int bucket) { - Preconditions.checkArgument(bucket >= 0); - - synchronized (this) { - count++; - double dev = bucket - mean; - mean += dev / count; - sumOfSquaredDeviation += dev * (bucket - mean); - } - - if (bucket >= bucketCounts.length) { - // Account for bucket overflow, records everything that is equals or greater of the last - // bucket. - bucketCounts[bucketCounts.length - 1].incrementAndGet(); - return; - } - - bucketCounts[bucket].incrementAndGet(); - } - - @Override - public String toString() { - ToStringHelper helper = MoreObjects.toStringHelper(Distribution.class); - helper.add("bucketCounts", bucketCounts); - helper.add("count", count); - helper.add("mean", mean); - helper.add("sumOfSquaredDeviation", sumOfSquaredDeviation); - return helper.toString(); - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/FlowController.java b/client/src/main/java/com/google/cloud/pubsub/FlowController.java deleted file mode 100644 index 6478289a..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/FlowController.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.cloud.pubsub.Publisher.CloudPubsubFlowControlException; -import com.google.cloud.pubsub.Publisher.MaxOutstandingBytesReachedException; -import com.google.cloud.pubsub.Publisher.MaxOutstandingMessagesReachedException; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import java.util.concurrent.Semaphore; -import javax.annotation.Nullable; - -/** Provides flow control capability for Pub/Sub client classes. */ -class FlowController { - @Nullable private final Semaphore outstandingMessageCount; - @Nullable private final Semaphore outstandingByteCount; - private final boolean failOnLimits; - private final Optional maxOutstandingMessages; - private final Optional maxOutstandingBytes; - - FlowController( - Optional maxOutstandingMessages, - Optional maxOutstandingBytes, - boolean failOnFlowControlLimits) { - this.maxOutstandingMessages = Preconditions.checkNotNull(maxOutstandingMessages); - this.maxOutstandingBytes = Preconditions.checkNotNull(maxOutstandingBytes); - outstandingMessageCount = - maxOutstandingMessages.isPresent() ? new Semaphore(maxOutstandingMessages.get()) : null; - outstandingByteCount = - maxOutstandingBytes.isPresent() ? new Semaphore(maxOutstandingBytes.get()) : null; - this.failOnLimits = failOnFlowControlLimits; - } - - void reserve(int messages, int bytes) throws CloudPubsubFlowControlException { - Preconditions.checkArgument(messages > 0); - - if (outstandingMessageCount != null) { - if (!failOnLimits) { - outstandingMessageCount.acquireUninterruptibly(messages); - } else if (!outstandingMessageCount.tryAcquire(messages)) { - throw new MaxOutstandingMessagesReachedException(maxOutstandingMessages.get()); - } - } - - // Will always allow to send a message even if it is larger than the flow control limit, - // if it doesn't then it will deadlock the thread. - if (outstandingByteCount != null) { - int permitsToDraw = Math.min(bytes, maxOutstandingBytes.get()); - if (!failOnLimits) { - outstandingByteCount.acquireUninterruptibly(permitsToDraw); - } else if (!outstandingByteCount.tryAcquire(permitsToDraw)) { - throw new MaxOutstandingBytesReachedException(maxOutstandingBytes.get()); - } - } - } - - void release(int messages, int bytes) { - Preconditions.checkArgument(messages > 0); - - if (outstandingMessageCount != null) { - outstandingMessageCount.release(messages); - } - if (outstandingByteCount != null) { - // Need to return at most as much bytes as it can be drawn. - int permitsToReturn = Math.min(bytes, maxOutstandingBytes.get()); - outstandingByteCount.release(permitsToReturn); - } - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/MessagesWaiter.java b/client/src/main/java/com/google/cloud/pubsub/MessagesWaiter.java deleted file mode 100644 index 97dd0363..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/MessagesWaiter.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.annotations.VisibleForTesting; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * A barrier kind of object that helps to keep track and synchronously wait on pending messages. - */ -class MessagesWaiter { - private int pendingMessages; - - MessagesWaiter() { - pendingMessages = 0; - } - - public synchronized void incrementPendingMessages(int messages) { - this.pendingMessages += messages; - if (pendingMessages == 0) { - notifyAll(); - } - } - - public synchronized void waitNoMessages() { - waitNoMessages(new AtomicBoolean()); - } - - @VisibleForTesting - synchronized void waitNoMessages(AtomicBoolean waitReached) { - boolean interrupted = false; - try { - while (pendingMessages > 0) { - try { - waitReached.set(true); - wait(); - } catch (InterruptedException e) { - // Ignored, uninterruptibly. - interrupted = true; - } - } - } finally { - if (interrupted) { - Thread.currentThread().interrupt(); - } - } - } - - @VisibleForTesting - public int pendingMessages() { - return pendingMessages; - } -} \ No newline at end of file diff --git a/client/src/main/java/com/google/cloud/pubsub/PollingSubscriberConnection.java b/client/src/main/java/com/google/cloud/pubsub/PollingSubscriberConnection.java deleted file mode 100644 index 1926097f..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/PollingSubscriberConnection.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import com.google.cloud.pubsub.Subscriber.MessageReceiver; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.GetSubscriptionRequest; -import com.google.pubsub.v1.ModifyAckDeadlineRequest; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.SubscriberGrpc; -import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; -import com.google.pubsub.v1.Subscription; -import io.grpc.Channel; -import io.grpc.StatusRuntimeException; -import io.grpc.auth.MoreCallCredentials; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of {@link AbstractSubscriberConnection} based on Cloud Pub/Sub pull and - * acknowledge operations. - */ -final class PollingSubscriberConnection extends AbstractSubscriberConnection { - private static final int MAX_PER_REQUEST_CHANGES = 1000; - private static final Duration DEFAULT_TIMEOUT = Duration.standardSeconds(10); - private static final int DEFAULT_MAX_MESSAGES = 1000; - private static final Duration INITIAL_BACKOFF = Duration.millis(100); // 100ms - private static final Duration MAX_BACKOFF = Duration.standardSeconds(10); // 10s - - private static final Logger logger = LoggerFactory.getLogger(PollingSubscriberConnection.class); - - private final SubscriberFutureStub stub; - - public PollingSubscriberConnection( - String subscription, - Credentials credentials, - MessageReceiver receiver, - Duration ackExpirationPadding, - Distribution ackLatencyDistribution, - Channel channel, - FlowController flowController, - ScheduledExecutorService executor) { - super( - subscription, - receiver, - ackExpirationPadding, - ackLatencyDistribution, - flowController, - executor); - stub = - SubscriberGrpc.newFutureStub(channel) - .withCallCredentials(MoreCallCredentials.from(credentials)); - } - - @Override - void initialize() { - ListenableFuture subscriptionInfo = - stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS) - .getSubscription( - GetSubscriptionRequest.newBuilder().setSubscription(subscription).build()); - - Futures.addCallback( - subscriptionInfo, - new FutureCallback() { - @Override - public void onSuccess(Subscription result) { - setMessageDeadlineSeconds(result.getAckDeadlineSeconds()); - pullMessages(INITIAL_BACKOFF); - } - - @Override - public void onFailure(Throwable cause) { - notifyFailed(cause); - } - }); - } - - private void pullMessages(final Duration backoff) { - ListenableFuture pullResult = - stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS) - .pull( - PullRequest.newBuilder() - .setSubscription(subscription) - .setMaxMessages(DEFAULT_MAX_MESSAGES) - .setReturnImmediately(true) - .build()); - - Futures.addCallback( - pullResult, - new FutureCallback() { - @Override - public void onSuccess(PullResponse pullResponse) { - processReceivedMessages(pullResponse.getReceivedMessagesList()); - if (pullResponse.getReceivedMessagesCount() == 0) { - // No messages in response, possibly caught up in backlog, we backoff to avoid - // slamming the server. - executor.schedule( - new Runnable() { - @Override - public void run() { - Duration newBackoff = backoff.multipliedBy(2); - if (newBackoff.isLongerThan(MAX_BACKOFF)) { - newBackoff = MAX_BACKOFF; - } - pullMessages(newBackoff); - } - }, - backoff.getMillis(), - TimeUnit.MILLISECONDS); - return; - } - pullMessages(INITIAL_BACKOFF); - } - - @Override - public void onFailure(Throwable cause) { - if (!(cause instanceof StatusRuntimeException) - || isRetryable(((StatusRuntimeException) cause).getStatus())) { - logger.error("Failed to pull messages (recoverable): " + cause.getMessage(), cause); - executor.schedule( - new Runnable() { - @Override - public void run() { - Duration newBackoff = backoff.multipliedBy(2); - if (newBackoff.isLongerThan(MAX_BACKOFF)) { - newBackoff = MAX_BACKOFF; - } - pullMessages(newBackoff); - } - }, - backoff.getMillis(), - TimeUnit.MILLISECONDS); - return; - } - notifyFailed(cause); - } - }); - } - - @Override - void sendAckOperations( - List acksToSend, List ackDeadlineExtensions) { - // Send the modify ack deadlines in batches as not to exceed the max request - // size. - List> modifyAckDeadlineChunks = - Lists.partition(ackDeadlineExtensions, MAX_PER_REQUEST_CHANGES); - for (List modAckChunk : modifyAckDeadlineChunks) { - for (PendingModifyAckDeadline modifyAckDeadline : modAckChunk) { - stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS) - .modifyAckDeadline( - ModifyAckDeadlineRequest.newBuilder() - .setSubscription(subscription) - .addAllAckIds(modifyAckDeadline.ackIds) - .setAckDeadlineSeconds(modifyAckDeadline.deadlineExtensionSeconds) - .build()); - } - } - - List> ackChunks = Lists.partition(acksToSend, MAX_PER_REQUEST_CHANGES); - Iterator> ackChunksIt = ackChunks.iterator(); - while (ackChunksIt.hasNext()) { - List ackChunk = ackChunksIt.next(); - stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS) - .acknowledge( - AcknowledgeRequest.newBuilder() - .setSubscription(subscription) - .addAllAckIds(ackChunk) - .build()); - } - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/Publisher.java b/client/src/main/java/com/google/cloud/pubsub/Publisher.java deleted file mode 100644 index 17a41796..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/Publisher.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannelBuilder; -import java.io.IOException; -import java.util.concurrent.ScheduledExecutorService; -import org.joda.time.Duration; - -/** - * A Cloud Pub/Sub publisher, that is - * associated with a specific topic at creation. - * - *

A {@link Publisher} provides built-in capabilities to automatically handle batching of - * messages, controlling memory utilization, and retrying API calls on transient errors. - * - *

With customizable options that control: - * - *

    - *
  • Message batching: such as number of messages or max batch byte size. - *
  • Flow control: such as max outstanding messages and maximum outstanding bytes. - *
  • Retries: such as the maximum duration of retries for a failing batch of messages. - *
- * - *

If no credentials are provided, the {@link Publisher} will use application default credentials - * through {@link GoogleCredentials#getApplicationDefault}. - * - *

For example, a {@link Publisher} can be constructed and used to publish a list of messages as - * follows: - * - * 

- *  Publisher publisher =
- *       Publisher.Builder.newBuilder(MY_TOPIC)
- *           .setMaxBatchDuration(new Duration(10 * 1000))
- *           .build();
- *  List> results = new ArrayList<>();
- *
- *  for (PubsubMessage messages : messagesToPublish) {
- *    results.add(publisher.publish(message));
- *  }
- *
- *  Futures.addCallback(
- *  Futures.allAsList(results),
- *  new FutureCallback>() {
- *    @Override
- *    public void onSuccess(List messageIds) {
- *      // ... process the acknowledgement of publish ...
- *    }
- *    @Override
- *    public void onFailure(Throwable t) {
- *      // .. handle the failure ...
- *    }
- *  });
- *
- *  // Ensure all the outstanding messages have been published before shutting down your process.
- *  publisher.shutdown();
- *
- */ -public interface Publisher { - String PUBSUB_API_ADDRESS = "pubsub.googleapis.com"; - String PUBSUB_API_SCOPE = "https://www.googleapis.com/auth/pubsub"; - - // API limits. - int MAX_BATCH_MESSAGES = 1000; - int MAX_BATCH_BYTES = 10 * 1000 * 1000; // 10 megabytes (https://en.wikipedia.org/wiki/Megabyte) - - // Meaningful defaults. - int DEFAULT_MAX_BATCH_MESSAGES = 100; - int DEFAULT_MAX_BATCH_BYTES = 1000; // 1 kB - Duration DEFAULT_MAX_BATCH_DURATION = new Duration(1); // 1ms - Duration DEFAULT_REQUEST_TIMEOUT = new Duration(10 * 1000); // 10 seconds - Duration MIN_SEND_BATCH_DURATION = new Duration(10 * 1000); // 10 seconds - Duration MIN_REQUEST_TIMEOUT = new Duration(10); // 10 milliseconds - - /** Topic to which the publisher publishes to. */ - String getTopic(); - - /** - * Schedules the publishing of a message. The publishing of the message may occur immediately or - * be delayed based on the publisher batching options. - * - *

Depending on chosen flow control {@link #failOnFlowControlLimits option}, the returned - * future might immediately fail with a {@link CloudPubsubFlowControlException} or block the - * current thread until there are more resources available to publish. - * - * @param message the message to publish. - * @return the message ID wrapped in a future. - */ - ListenableFuture publish(PubsubMessage message); - - /** Maximum amount of time to wait until scheduling the publishing of messages. */ - Duration getMaxBatchDuration(); - - /** Maximum number of bytes to batch before publishing. */ - long getMaxBatchBytes(); - - /** Maximum number of messages to batch before publishing. */ - long getMaxBatchMessages(); - - /** - * Maximum number of outstanding (i.e. pending to publish) messages before limits are enforced. - * See {@link #failOnFlowControlLimits()}. - */ - Optional getMaxOutstandingMessages(); - - /** - * Maximum number of outstanding (i.e. pending to publish) bytes before limits are enforced. See - * {@link #failOnFlowControlLimits()}. - */ - Optional getMaxOutstandingBytes(); - - /** - * Whether to block publish calls when reaching flow control limits (see {@link - * #getMaxOutstandingBytes()} & {@link #getMaxOutstandingMessages()}). - * - *

If set to false, a publish call will fail with either {@link - * MaxOutstandingBytesReachedException} or {@link MaxOutstandingMessagesReachedException}, as - * appropriate, when flow control limits are reached. - */ - boolean failOnFlowControlLimits(); - - /** Retrieves a snapshot of the publisher current {@link PublisherStats statistics}. */ - PublisherStats getStats(); - - /** - * Schedules immediate publishing of any outstanding messages and waits until all are processed. - * - *

Sends remaining outstanding messages and prevents future calls to publish. This method - * should be invoked prior to deleting the {@link Publisher} object in order to ensure that no - * pending messages are lost. - */ - void shutdown(); - - /** A builder of {@link Publisher}s. */ - final class Builder { - String topic; - - // Batching options - int maxBatchMessages; - int maxBatchBytes; - Duration maxBatchDuration; - - // Client-side flow control options - Optional maxOutstandingMessages; - Optional maxOutstandingBytes; - boolean failOnFlowControlLimits; - - // Send batch deadline - Duration sendBatchDeadline; - - // RPC options - Duration requestTimeout; - - // Channels and credentials - Optional userCredentials; - Optional>> channelBuilder; - - Optional executor; - - /** Constructs a new {@link Builder} using the given topic. */ - public static Builder newBuilder(String topic) { - return new Builder(topic); - } - - Builder(String topic) { - this.topic = Preconditions.checkNotNull(topic); - setDefaults(); - } - - private void setDefaults() { - userCredentials = Optional.absent(); - channelBuilder = Optional.absent(); - maxOutstandingMessages = Optional.absent(); - maxOutstandingBytes = Optional.absent(); - maxBatchMessages = DEFAULT_MAX_BATCH_MESSAGES; - maxBatchBytes = DEFAULT_MAX_BATCH_BYTES; - maxBatchDuration = DEFAULT_MAX_BATCH_DURATION; - requestTimeout = DEFAULT_REQUEST_TIMEOUT; - sendBatchDeadline = MIN_SEND_BATCH_DURATION; - failOnFlowControlLimits = false; - executor = Optional.absent(); - } - - /** - * Credentials to authenticate with. - * - *

Must be properly scoped for accessing Cloud Pub/Sub APIs. - */ - public Builder setCredentials(Credentials userCredentials) { - this.userCredentials = Optional.of(Preconditions.checkNotNull(userCredentials)); - return this; - } - - /** - * ManagedChannelBuilder to use to create Channels. - * - *

Must point at Cloud Pub/Sub endpoint. - */ - public Builder setChannelBuilder( - ManagedChannelBuilder> channelBuilder) { - this.channelBuilder = - Optional.>>of( - Preconditions.checkNotNull(channelBuilder)); - return this; - } - - // Batching options - - /** - * Maximum number of messages to send per publish call. - * - *

It also sets a target to when to trigger a publish. - */ - public Builder setMaxBatchMessages(int messages) { - Preconditions.checkArgument(messages > 0); - maxBatchMessages = messages; - return this; - } - - /** - * Maximum number of bytes to send per publish call. - * - *

It also sets a target to when to trigger a publish. - * - *

This will not be honored if a single message is published that exceeds this maximum. - */ - public Builder setMaxBatchBytes(int bytes) { - Preconditions.checkArgument(bytes > 0); - maxBatchBytes = bytes; - return this; - } - - /** - * Time to wait, since the first message is kept in memory for batching, before triggering a - * publish call. - */ - public Builder setMaxBatchDuration(Duration duration) { - Preconditions.checkArgument(duration.getMillis() >= 0); - maxBatchDuration = duration; - return this; - } - - // Flow control options - - /** Maximum number of outstanding messages to keep in memory before enforcing flow control. */ - public Builder setMaxOutstandingMessages(int messages) { - Preconditions.checkArgument(messages > 0); - maxOutstandingMessages = Optional.of(messages); - return this; - } - - /** Maximum number of outstanding messages to keep in memory before enforcing flow control. */ - public Builder setMaxOutstandingBytes(int bytes) { - Preconditions.checkArgument(bytes > 0); - maxOutstandingBytes = Optional.of(bytes); - return this; - } - - /** - * Whether to fail publish when reaching any of the flow control limits, with either a {@link - * MaxOutstandingBytesReachedException} or {@link MaxOutstandingMessagesReachedException} as - * appropriate. - * - *

If set to false, then publish operations will block the current thread until the - * outstanding requests go under the limits. - */ - public Builder setFailOnFlowControlLimits(boolean fail) { - failOnFlowControlLimits = fail; - return this; - } - - /** Maximum time to attempt sending (and retrying) a batch of messages before giving up. */ - public Builder setSendBatchDeadline(Duration deadline) { - Preconditions.checkArgument(deadline.compareTo(MIN_SEND_BATCH_DURATION) >= 0); - sendBatchDeadline = deadline; - return this; - } - - // Runtime options - /** Time to wait for a publish call to return from the server. */ - public Builder setRequestTimeout(Duration timeout) { - Preconditions.checkArgument(timeout.compareTo(MIN_REQUEST_TIMEOUT) >= 0); - requestTimeout = timeout; - return this; - } - - /** Gives the ability to set a custom executor to be used by the library. */ - public Builder setExecutor(ScheduledExecutorService executor) { - this.executor = Optional.of(Preconditions.checkNotNull(executor)); - return this; - } - - public Publisher build() throws IOException { - return new PublisherImpl(this); - } - } - - /** Base exception that signals a flow control state. */ - abstract class CloudPubsubFlowControlException extends Exception {} - - /** - * Returned as a future exception when client-side flow control is enforced based on the maximum - * number of outstanding in-memory messages. - */ - final class MaxOutstandingMessagesReachedException extends CloudPubsubFlowControlException { - private final int currentMaxMessages; - - public MaxOutstandingMessagesReachedException(int currentMaxMessages) { - this.currentMaxMessages = currentMaxMessages; - } - - public int getCurrentMaxBatchMessages() { - return currentMaxMessages; - } - - @Override - public String toString() { - return String.format( - "The maximum number of batch messages: %d have been reached.", currentMaxMessages); - } - } - - /** - * Returned as a future exception when client-side flow control is enforced based on the maximum - * number of unacknowledged in-memory bytes. - */ - final class MaxOutstandingBytesReachedException extends CloudPubsubFlowControlException { - private final int currentMaxBytes; - - public MaxOutstandingBytesReachedException(int currentMaxBytes) { - this.currentMaxBytes = currentMaxBytes; - } - - public int getCurrentMaxBatchBytes() { - return currentMaxBytes; - } - - @Override - public String toString() { - return String.format( - "The maximum number of batch bytes: %d have been reached.", currentMaxBytes); - } - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/PublisherImpl.java b/client/src/main/java/com/google/cloud/pubsub/PublisherImpl.java deleted file mode 100644 index bb9bc1ae..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/PublisherImpl.java +++ /dev/null @@ -1,452 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableList; -import com.google.common.primitives.Ints; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.Status; -import io.grpc.auth.MoreCallCredentials; -import io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.NegotiationType; -import io.grpc.netty.NettyChannelBuilder; -import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Implementation of {@link Publisher}. */ -final class PublisherImpl implements Publisher { - private static final int DEFAULT_MIN_THREAD_POOL_SIZE = 5; - private static final double INITIAL_BACKOFF_MS = 5; - private static final double BACKOFF_RANDOMNESS_FACTOR = 0.2; - - private static final Logger logger = LoggerFactory.getLogger(PublisherImpl.class); - - private final String topic; - - private final int maxBatchMessages; - private final int maxBatchBytes; - private final Duration maxBatchDuration; - private final boolean hasBatchingBytes; - - private final Optional maxOutstandingMessages; - private final Optional maxOutstandingBytes; - private final boolean failOnFlowControlLimits; - - private final Lock messagesBatchLock; - private List messagesBatch; - private int batchedBytes; - - private final AtomicBoolean activeAlarm; - - private final FlowController flowController; - private final Channel[] channels; - private final AtomicLong channelIndex; - private final CallCredentials credentials; - private final Duration requestTimeout; - - private final ScheduledExecutorService executor; - private final AtomicBoolean shutdown; - private final MessagesWaiter messagesWaiter; - private final Duration sendBatchDeadline; - private ScheduledFuture currentAlarmFuture; - - PublisherImpl(Builder builder) throws IOException { - topic = builder.topic; - - maxBatchMessages = builder.maxBatchMessages; - maxBatchBytes = builder.maxBatchBytes; - maxBatchDuration = builder.maxBatchDuration; - hasBatchingBytes = maxBatchBytes > 0; - - maxOutstandingMessages = builder.maxOutstandingMessages; - maxOutstandingBytes = builder.maxOutstandingBytes; - failOnFlowControlLimits = builder.failOnFlowControlLimits; - this.flowController = - new FlowController(maxOutstandingMessages, maxOutstandingBytes, failOnFlowControlLimits); - - sendBatchDeadline = builder.sendBatchDeadline; - - requestTimeout = builder.requestTimeout; - - messagesBatch = new LinkedList<>(); - messagesBatchLock = new ReentrantLock(); - activeAlarm = new AtomicBoolean(false); - int numCores = Math.max(1, Runtime.getRuntime().availableProcessors()); - executor = - builder.executor.isPresent() - ? builder.executor.get() - : Executors.newScheduledThreadPool( - numCores * DEFAULT_MIN_THREAD_POOL_SIZE, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("cloud-pubsub-publisher-thread-%d") - .build()); - channels = new Channel[numCores]; - channelIndex = new AtomicLong(0); - for (int i = 0; i < numCores; i++) { - channels[i] = - builder.channelBuilder.isPresent() - ? builder.channelBuilder.get().build() - : NettyChannelBuilder.forAddress(PUBSUB_API_ADDRESS, 443) - .negotiationType(NegotiationType.TLS) - .sslContext(GrpcSslContexts.forClient().ciphers(null).build()) - .executor(executor) - .build(); - } - credentials = - MoreCallCredentials.from( - builder.userCredentials.isPresent() - ? builder.userCredentials.get() - : GoogleCredentials.getApplicationDefault() - .createScoped(Collections.singletonList(PUBSUB_API_SCOPE))); - shutdown = new AtomicBoolean(false); - messagesWaiter = new MessagesWaiter(); - } - - @Override - public PublisherStats getStats() { - // TODO: Implement this. - throw new UnsupportedOperationException(); - } - - @Override - public Duration getMaxBatchDuration() { - return maxBatchDuration; - } - - @Override - public long getMaxBatchBytes() { - return maxBatchBytes; - } - - @Override - public long getMaxBatchMessages() { - return maxBatchMessages; - } - - @Override - public Optional getMaxOutstandingMessages() { - return maxOutstandingMessages; - } - - @Override - public Optional getMaxOutstandingBytes() { - return maxOutstandingBytes; - } - - @Override - public boolean failOnFlowControlLimits() { - return failOnFlowControlLimits; - } - - /** Whether flow control kicks in on a per outstanding messages basis. */ - boolean isPerMessageEnforced() { - return maxOutstandingMessages.isPresent(); - } - - /** Whether flow control kicks in on a per outstanding bytes basis. */ - boolean isPerBytesEnforced() { - return maxOutstandingBytes.isPresent(); - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public ListenableFuture publish(PubsubMessage message) { - if (shutdown.get()) { - throw new IllegalStateException("Cannot publish on a shut-down publisher."); - } - - final int messageSize = message.getSerializedSize(); - try { - flowController.reserve(1, messageSize); - } catch (CloudPubsubFlowControlException e) { - return Futures.immediateFailedFuture(e); - } - OutstandingBatch batchToSend = null; - SettableFuture publishResult = SettableFuture.create(); - final OutstandingPublish outstandingPublish = new OutstandingPublish(publishResult, message); - messagesBatchLock.lock(); - try { - // Check if the next message makes the batch exceed the current batch byte size. - if (!messagesBatch.isEmpty() - && hasBatchingBytes - && batchedBytes + messageSize >= getMaxBatchBytes()) { - batchToSend = new OutstandingBatch(messagesBatch, batchedBytes); - messagesBatch = new LinkedList<>(); - batchedBytes = 0; - } - - // Border case if the message to send is greater equals to the max batch size then can't be - // included in the current batch and instead sent immediately. - if (!hasBatchingBytes || messageSize < getMaxBatchBytes()) { - batchedBytes += messageSize; - messagesBatch.add(outstandingPublish); - - // If after adding the message we have reached the batch max messages then we have a batch - // to send. - if (messagesBatch.size() == getMaxBatchMessages()) { - batchToSend = new OutstandingBatch(messagesBatch, batchedBytes); - messagesBatch = new LinkedList<>(); - batchedBytes = 0; - } - } - // Setup the next duration based delivery alarm if there are messages batched. - if (!messagesBatch.isEmpty()) { - setupDurationBasedPublishAlarm(); - } else if (currentAlarmFuture != null) { - logger.debug("Cancelling alarm"); - if (activeAlarm.getAndSet(false)) { - currentAlarmFuture.cancel(false); - } - } - } finally { - messagesBatchLock.unlock(); - } - - messagesWaiter.incrementPendingMessages(1); - - if (batchToSend != null) { - logger.debug("Scheduling a batch for immediate sending."); - final OutstandingBatch finalBatchToSend = batchToSend; - executor.execute( - new Runnable() { - @Override - public void run() { - publishOutstandingBatch(finalBatchToSend); - } - }); - } - - // If the message is over the size limit, it was not added to the pending messages and it will - // be sent in its own batch immediately. - if (hasBatchingBytes && messageSize >= getMaxBatchBytes()) { - logger.debug("Message exceeds the max batch bytes, scheduling it for immediate send."); - executor.execute( - new Runnable() { - @Override - public void run() { - publishOutstandingBatch( - new OutstandingBatch(ImmutableList.of(outstandingPublish), messageSize)); - } - }); - } - - return publishResult; - } - - private void setupDurationBasedPublishAlarm() { - if (!activeAlarm.getAndSet(true)) { - logger.debug("Setting up alarm for the next %d ms.", getMaxBatchDuration().getMillis()); - currentAlarmFuture = - executor.schedule( - new Runnable() { - @Override - public void run() { - logger.debug("Sending messages based on schedule."); - activeAlarm.getAndSet(false); - publishAllOustanding(); - } - }, - getMaxBatchDuration().getMillis(), - TimeUnit.MILLISECONDS); - } - } - - private void publishAllOustanding() { - messagesBatchLock.lock(); - OutstandingBatch batchToSend; - try { - if (messagesBatch.isEmpty()) { - return; - } - batchToSend = new OutstandingBatch(messagesBatch, batchedBytes); - messagesBatch = new LinkedList<>(); - batchedBytes = 0; - } finally { - messagesBatchLock.unlock(); - } - publishOutstandingBatch(batchToSend); - } - - private void publishOutstandingBatch(final OutstandingBatch outstandingBatch) { - PublishRequest.Builder publishRequest = PublishRequest.newBuilder(); - publishRequest.setTopic(topic); - for (OutstandingPublish outstandingPublish : outstandingBatch.outstandingPublishes) { - publishRequest.addMessages(outstandingPublish.message); - } - int currentChannel = (int) (channelIndex.getAndIncrement() % channels.length); - Futures.addCallback( - PublisherGrpc.newFutureStub(channels[currentChannel]) - .withCallCredentials(credentials) - .withDeadlineAfter(requestTimeout.getMillis(), TimeUnit.MILLISECONDS) - .publish(publishRequest.build()), - new FutureCallback() { - @Override - public void onSuccess(PublishResponse result) { - try { - if (result.getMessageIdsCount() != outstandingBatch.size()) { - Throwable t = - new IllegalStateException( - String.format( - "The publish result count %s does not match " - + "the expected %s results. Please contact Cloud Pub/Sub support " - + "if this frequently occurs", - result.getMessageIdsCount(), outstandingBatch.size())); - for (OutstandingPublish oustandingMessage : outstandingBatch.outstandingPublishes) { - oustandingMessage.publishResult.setException(t); - } - return; - } - - Iterator messagesResultsIt = - outstandingBatch.outstandingPublishes.iterator(); - for (String messageId : result.getMessageIdsList()) { - messagesResultsIt.next().publishResult.set(messageId); - } - } finally { - flowController.release(outstandingBatch.size(), outstandingBatch.batchSizeBytes); - messagesWaiter.incrementPendingMessages(-outstandingBatch.size()); - } - } - - @Override - public void onFailure(Throwable t) { - long nextBackoffDelay = computeNextBackoffDelayMs(outstandingBatch); - - if (!isRetryable(t) - || System.currentTimeMillis() + nextBackoffDelay - > outstandingBatch.creationTime - + PublisherImpl.this.sendBatchDeadline.getMillis()) { - try { - for (OutstandingPublish outstandingPublish : - outstandingBatch.outstandingPublishes) { - outstandingPublish.publishResult.setException(t); - } - } finally { - messagesWaiter.incrementPendingMessages(-outstandingBatch.size()); - } - return; - } - - executor.schedule( - new Runnable() { - @Override - public void run() { - publishOutstandingBatch(outstandingBatch); - } - }, - nextBackoffDelay, - TimeUnit.MILLISECONDS); - } - }); - } - - private static final class OutstandingBatch { - final List outstandingPublishes; - final long creationTime; - int attempt; - int batchSizeBytes; - - OutstandingBatch(List outstandingPublishes, int batchSizeBytes) { - this.outstandingPublishes = outstandingPublishes; - attempt = 1; - creationTime = System.currentTimeMillis(); - this.batchSizeBytes = batchSizeBytes; - } - - public int size() { - return outstandingPublishes.size(); - } - } - - private static final class OutstandingPublish { - SettableFuture publishResult; - PubsubMessage message; - - OutstandingPublish(SettableFuture publishResult, PubsubMessage message) { - this.publishResult = publishResult; - this.message = message; - } - } - - @Override - public void shutdown() { - if (shutdown.getAndSet(true)) { - throw new IllegalStateException("Cannot shut down a publisher already shut-down."); - } - if (currentAlarmFuture != null && activeAlarm.getAndSet(false)) { - currentAlarmFuture.cancel(false); - } - publishAllOustanding(); - messagesWaiter.waitNoMessages(); - } - - private static long computeNextBackoffDelayMs(OutstandingBatch outstandingBatch) { - long delayMillis = Math.round(Math.scalb(INITIAL_BACKOFF_MS, outstandingBatch.attempt)); - int randomWaitMillis = - Ints.saturatedCast( - (long) ((Math.random() - 0.5) * 2 * delayMillis * BACKOFF_RANDOMNESS_FACTOR)); - ++outstandingBatch.attempt; - return delayMillis + randomWaitMillis; - } - - private boolean isRetryable(Throwable t) { - Status status = Status.fromThrowable(t); - switch (status.getCode()) { - case ABORTED: - case CANCELLED: - case DEADLINE_EXCEEDED: - case INTERNAL: - case RESOURCE_EXHAUSTED: - case UNKNOWN: - case UNAVAILABLE: - return true; - default: - return false; - } - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/PublisherStats.java b/client/src/main/java/com/google/cloud/pubsub/PublisherStats.java deleted file mode 100644 index 4d6db216..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/PublisherStats.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import javax.annotation.concurrent.Immutable; - -/** - * A snapshot of the publisher statistics at the time they were requested from the {@link - * Publisher}. - */ -//TODO: Finish implementation. -@Immutable -public class PublisherStats { - private final long sentMessages; - private final long ackedMessages; - private final long failedMessages; - private final long pendingMessages; - - PublisherStats(long sentMessages, long ackedMessages, long failedMessages, long pendingMessages) { - this.sentMessages = sentMessages; - this.ackedMessages = ackedMessages; - this.failedMessages = failedMessages; - this.pendingMessages = pendingMessages; - } - - /** Number of successfully published messages. */ - public long getAckedMessages() { - return ackedMessages; - } - - /** Number of messages that failed to publish. */ - public long getFailedMessages() { - return failedMessages; - } - - /** Number of messages pending to publish, includes message in-flight. */ - public long getPendingMessages() { - return pendingMessages; - } - - /** Total messages sent, equal to pending + acked + failed messages. */ - public long getSentMessages() { - return sentMessages; - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/StreamingSubscriberConnection.java b/client/src/main/java/com/google/cloud/pubsub/StreamingSubscriberConnection.java deleted file mode 100644 index 5aeb0524..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/StreamingSubscriberConnection.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import com.google.cloud.pubsub.Subscriber.MessageReceiver; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.SettableFuture; -import com.google.pubsub.v1.StreamingPullRequest; -import com.google.pubsub.v1.StreamingPullResponse; -import com.google.pubsub.v1.SubscriberGrpc; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.Status; -import io.grpc.auth.MoreCallCredentials; -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.ClientCalls; -import io.grpc.stub.ClientResponseObserver; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Implementation of {@link AbstractSubscriberConnection} based on Cloud Pub/Sub streaming pull. */ -final class StreamingSubscriberConnection extends AbstractSubscriberConnection { - private static final Logger logger = LoggerFactory.getLogger(StreamingSubscriberConnection.class); - - private static final Duration INITIAL_CHANNEL_RECONNECT_BACKOFF = new Duration(100); // 100ms - private static final int MAX_PER_REQUEST_CHANGES = 10000; - - private Duration channelReconnectBackoff = INITIAL_CHANNEL_RECONNECT_BACKOFF; - - private final Channel channel; - private final Credentials credentials; - - private ClientCallStreamObserver requestObserver; - - public StreamingSubscriberConnection( - String subscription, - Credentials credentials, - MessageReceiver receiver, - Duration ackExpirationPadding, - int streamAckDeadlineSeconds, - Distribution ackLatencyDistribution, - Channel channel, - FlowController flowController, - ScheduledExecutorService executor) { - super( - subscription, - receiver, - ackExpirationPadding, - ackLatencyDistribution, - flowController, - executor); - this.credentials = credentials; - this.channel = channel; - setMessageDeadlineSeconds(streamAckDeadlineSeconds); - } - - @Override - protected void doStop() { - super.doStop(); - requestObserver.onError(Status.CANCELLED.asException()); - } - - @Override - void initialize() { - final SettableFuture errorFuture = SettableFuture.create(); - final ClientResponseObserver responseObserver = - new ClientResponseObserver() { - @Override - public void beforeStart(ClientCallStreamObserver requestObserver) { - StreamingSubscriberConnection.this.requestObserver = requestObserver; - requestObserver.disableAutoInboundFlowControl(); - } - - @Override - public void onNext(StreamingPullResponse response) { - processReceivedMessages(response.getReceivedMessagesList()); - // Only if not shutdown we will request one more batch of messages to be delivered. - if (isAlive()) { - requestObserver.request(1); - } - } - - @Override - public void onError(Throwable t) { - logger.debug("Terminated streaming with exception", t); - errorFuture.setException(t); - } - - @Override - public void onCompleted() { - logger.debug("Streaming pull terminated successfully!"); - errorFuture.set(null); - } - }; - final ClientCallStreamObserver requestObserver = - (ClientCallStreamObserver) - (ClientCalls.asyncBidiStreamingCall( - channel.newCall( - SubscriberGrpc.METHOD_STREAMING_PULL, - CallOptions.DEFAULT.withCallCredentials(MoreCallCredentials.from(credentials))), - responseObserver)); - logger.debug( - "Initializing stream to subscription {} with deadline {}", - subscription, - getMessageDeadlineSeconds()); - requestObserver.onNext( - StreamingPullRequest.newBuilder() - .setSubscription(subscription) - .setStreamAckDeadlineSeconds(getMessageDeadlineSeconds()) - .build()); - requestObserver.request(1); - - Futures.addCallback( - errorFuture, - new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - channelReconnectBackoff = INITIAL_CHANNEL_RECONNECT_BACKOFF; - // The stream was closed. And any case we want to reopen it to continue receiving - // messages. - initialize(); - } - - @Override - public void onFailure(Throwable t) { - Status errorStatus = Status.fromThrowable(t); - if (isRetryable(errorStatus) && isAlive()) { - long backoffMillis = channelReconnectBackoff.getMillis(); - channelReconnectBackoff = channelReconnectBackoff.plus(backoffMillis); - executor.schedule( - new Runnable() { - @Override - public void run() { - initialize(); - } - }, - backoffMillis, - TimeUnit.MILLISECONDS); - } else { - if (isAlive()) { - notifyFailed(t); - } - } - } - }, - executor); - } - - private boolean isAlive() { - return state() == State.RUNNING || state() == State.STARTING; - } - - @Override - void sendAckOperations( - List acksToSend, List ackDeadlineExtensions) { - - // Send the modify ack deadlines in batches as not to exceed the max request - // size. - List> ackChunks = Lists.partition(acksToSend, MAX_PER_REQUEST_CHANGES); - List> modifyAckDeadlineChunks = - Lists.partition(ackDeadlineExtensions, MAX_PER_REQUEST_CHANGES); - Iterator> ackChunksIt = ackChunks.iterator(); - Iterator> modifyAckDeadlineChunksIt = - modifyAckDeadlineChunks.iterator(); - - while (ackChunksIt.hasNext() || modifyAckDeadlineChunksIt.hasNext()) { - com.google.pubsub.v1.StreamingPullRequest.Builder requestBuilder = - StreamingPullRequest.newBuilder(); - if (modifyAckDeadlineChunksIt.hasNext()) { - List modAckChunk = modifyAckDeadlineChunksIt.next(); - for (PendingModifyAckDeadline modifyAckDeadline : modAckChunk) { - for (String ackId : modifyAckDeadline.ackIds) { - requestBuilder.addModifyDeadlineSeconds(modifyAckDeadline.deadlineExtensionSeconds) - .addModifyDeadlineAckIds(ackId); - } - } - } - if (ackChunksIt.hasNext()) { - List ackChunk = ackChunksIt.next(); - requestBuilder.addAllAckIds(ackChunk); - } - requestObserver.onNext(requestBuilder.build()); - } - } - - public void updateStreamAckDeadline(int newAckDeadlineSeconds) { - setMessageDeadlineSeconds(newAckDeadlineSeconds); - requestObserver.onNext( - StreamingPullRequest.newBuilder() - .setStreamAckDeadlineSeconds(newAckDeadlineSeconds) - .build()); - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/Subscriber.java b/client/src/main/java/com/google/cloud/pubsub/Subscriber.java deleted file mode 100644 index c7f78202..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/Subscriber.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.pubsub.Subscriber.MessageReceiver.AckReply; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.Service; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.ManagedChannelBuilder; -import java.io.IOException; -import java.util.concurrent.ScheduledExecutorService; -import org.joda.time.Duration; - -/** - * A Cloud Pub/Sub subscriber that is - * associated with a specific subscription at creation. - * - *

A {@link Subscriber} allows you to provide an implementation of a {@link MessageReceiver - * receiver} to which messages are going to be delivered as soon as they are received by the - * subscriber. The delivered messages then can be {@link AckReply#ACK acked} or {@link - * AckReply#NACK nacked} at will as they get processed by the receiver. Nacking a - * messages implies a later redelivery of such message. - * - *

The subscriber handles the ack management, by automatically extending the ack deadline while - * the message is being processed, to then issue the ack or nack of such message when the processing - * is done. Note: message redelivery is still possible. - * - *

It also provides customizable options that control: - * - *

    - *
  • Ack deadline extension: such as the amount of time ahead to trigger the extension of - * message acknowledgement expiration. - *
  • Flow control: such as the maximum outstanding messages or maximum outstanding bytes to keep - * in memory before the receiver either ack or nack them. - *
- * - *

If no credentials are provided, the {@link Publisher} will use application default - * credentials through {@link GoogleCredentials#getApplicationDefault}. - * - *

For example, a {@link Subscriber} can be constructed and used to receive messages as follows: - * - * 

- *  MessageReceiver receiver =
- *      message -> {
- *        // ... process message ...
- *        return Futures.immediateFuture(AckReply.ACK);
- *      });
- *
- *  Subscriber subscriber =
- *      Subscriber.Builder.newBuilder(MY_SUBSCRIPTION, receiver)
- *          .setMaxBatchAcks(100)
- *          .build();
- *
- *  subscriber.startAsync();
- *
- *  ... recommended, listen for fatal errors that break the subscriber streaming ...
- *  subscriber.addListener(
-        new Listener() {
-          @Override
-          public void failed(State from, Throwable failure) {
-            System.out.println("Subscriber faile with error: " + failure);
-          }
-        },
-        Executors.newSingleThreadExecutor());
- *
- *  ... and when done with the subscriber ...
- *  subscriber.stopAsync();
- *
- */ -public interface Subscriber extends Service { - String PUBSUB_API_ADDRESS = "pubsub.googleapis.com"; - String PUBSUB_API_SCOPE = "https://www.googleapis.com/auth/pubsub"; - - /** Retrieves a snapshot of the current subscriber statistics. */ - SubscriberStats getStats(); - - /** Users of the {@link Subscriber} must implement this interface to receive messages. */ - interface MessageReceiver { - public static enum AckReply { - /** - * To be used for acking a message. - */ - ACK, - /** - * To be used for nacking a message. - */ - NACK - } - - /** - * Called when a message is received by the subscriber. - * - * @return A future that signals when a message has been processed. - */ - ListenableFuture receiveMessage(PubsubMessage message); - } - - /** Subscription for which the subscriber is streaming messages. */ - String getSubscription(); - - /** - * Time before a message is to expire when the subscriber is going to attempt to renew its ack - * deadline. - */ - Duration getAckExpirationPadding(); - - /** - * Maximum number of outstanding (i.e. pending to process) messages before limits are enforced. - * - *

When limits are enforced, no more messages will be dispatched to the {@link - * MessageReceiver} but due to the gRPC and HTTP/2 buffering and congestion control window - * management, still some extra bytes could be kept at lower layers. - */ - Optional getMaxOutstandingMessages(); - - /** - * Maximum number of outstanding (i.e. pending to process) bytes before limits are enforced. - */ - Optional getMaxOutstandingBytes(); - - /** Builder of {@link Subscriber Subscribers}. */ - final class Builder { - private static final Duration MIN_ACK_EXPIRATION_PADDING = Duration.millis(100); - private static final Duration DEFAULT_ACK_EXPIRATION_PADDING = Duration.millis(500); - - String subscription; - Optional credentials; - MessageReceiver receiver; - - Duration ackExpirationPadding; - - Optional maxOutstandingMessages; - Optional maxOutstandingBytes; - - Optional executor; - Optional>> channelBuilder; - - /** - * Constructs a new {@link Builder}. - * - *

Once {@link #build()} is called a gRPC stub will be created for use of the {@link - * Publisher}. - * - * @param subscription Cloud Pub/Sub subscription to bind the subscriber to - * @param receiver an implementation of {@link MessageReceiver} used to process the received - * messages - */ - public static Builder newBuilder(String subscription, MessageReceiver receiver) { - return new Builder(subscription, receiver); - } - - Builder(String subscription, MessageReceiver receiver) { - setDefaults(); - this.subscription = subscription; - this.receiver = receiver; - } - - private void setDefaults() { - credentials = Optional.absent(); - channelBuilder = Optional.absent(); - ackExpirationPadding = DEFAULT_ACK_EXPIRATION_PADDING; - maxOutstandingBytes = Optional.absent(); - maxOutstandingMessages = Optional.absent(); - executor = Optional.absent(); - } - - /** - * Credentials to authenticate with. - * - *

Must be properly scoped for accessing Cloud Pub/Sub APIs. - */ - public Builder setCredentials(Credentials credentials) { - this.credentials = Optional.of(Preconditions.checkNotNull(credentials)); - return this; - } - - /** - * ManagedChannelBuilder to use to create Channels. - * - *

Must point at Cloud Pub/Sub endpoint. - */ - public Builder setChannelBuilder( - ManagedChannelBuilder> channelBuilder) { - this.channelBuilder = - Optional.>>of( - Preconditions.checkNotNull(channelBuilder)); - return this; - } - - /** - * Sets the maximum number of outstanding messages; messages delivered to the {@link - * MessageReceiver} that have not been acknowledged or rejected. - * - * @param maxOutstandingMessages must be greater than 0 - */ - public Builder setMaxOutstandingMessages(int maxOutstandingMessages) { - Preconditions.checkArgument( - maxOutstandingMessages > 0, - "maxOutstandingMessages limit is disabled by default, but if set it must be set to a " - + "value greater to 0."); - this.maxOutstandingMessages = Optional.of(maxOutstandingMessages); - return this; - } - - /** - * Sets the maximum number of outstanding bytes; bytes delivered to the {@link MessageReceiver} - * that have not been acknowledged or rejected. - * - * @param maxOutstandingBytes must be greater than 0 - */ - public Builder setMaxOutstandingBytes(int maxOutstandingBytes) { - Preconditions.checkArgument( - maxOutstandingBytes > 0, - "maxOutstandingBytes limit is disabled by default, but if set it must be set to a value " - + "greater than 0."); - this.maxOutstandingBytes = Optional.of(maxOutstandingBytes); - return this; - } - - /** - * Set acknowledgement expiration padding. - * - *

This is the time accounted before a message expiration is to happen, so the - * {@link Subscriber} is able to send an ack extension beforehand. - * - *

This padding duration is configurable so you can account for network latency. A reasonable - * number must be provided so messages don't expire because of network latency between when the - * ack extension is required and when it reaches the Pub/Sub service. - * - * @param ackExpirationPadding must be greater or equal to {@link #MIN_ACK_EXPIRATION_PADDING} - */ - public Builder setAckExpirationPadding(Duration ackExpirationPadding) { - Preconditions.checkArgument(ackExpirationPadding.compareTo(MIN_ACK_EXPIRATION_PADDING) >= 0); - this.ackExpirationPadding = ackExpirationPadding; - return this; - } - - /** Gives the ability to set a custom executor. */ - public Builder setExecutor(ScheduledExecutorService executor) { - this.executor = Optional.of(executor); - return this; - } - - public Subscriber build() throws IOException { - return new SubscriberImpl(this); - } - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/SubscriberImpl.java b/client/src/main/java/com/google/cloud/pubsub/SubscriberImpl.java deleted file mode 100644 index 935bbea8..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/SubscriberImpl.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; -import com.google.common.primitives.Ints; -import com.google.common.util.concurrent.AbstractService; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.NegotiationType; -import io.grpc.netty.NettyChannelBuilder; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Implementation of {@link Subscriber}. */ -public class SubscriberImpl extends AbstractService implements Subscriber { - private static final int THREADS_PER_CHANNEL = 5; - @VisibleForTesting static final int CHANNELS_PER_CORE = 10; - private static final int MAX_INBOUND_MESSAGE_SIZE = - 20 * 1024 * 1024; // 20MB API maximum message size. - private static final int INITIAL_ACK_DEADLINE_SECONDS = 10; - private static final int MAX_ACK_DEADLINE_SECONDS = 600; - private static final int MIN_ACK_DEADLINE_SECONDS = 10; - private static final Duration ACK_DEADLINE_UPDATE_PERIOD = Duration.standardMinutes(1); - private static final double PERCENTILE_FOR_ACK_DEADLINE_UPDATES = 99.9; - - private static final Logger logger = LoggerFactory.getLogger(SubscriberImpl.class); - - private final String subscription; - private final Optional maxOutstandingBytes; - private final Optional maxOutstandingMessages; - private final Duration ackExpirationPadding; - private final ScheduledExecutorService executor; - private final Distribution ackLatencyDistribution = - new Distribution(MAX_ACK_DEADLINE_SECONDS + 1); - private final int numChannels; - private final FlowController flowController; - private final ManagedChannelBuilder> channelBuilder; - private final Credentials credentials; - private final MessageReceiver receiver; - private final List streamingSubscriberConnections; - private final List pollingSubscriberConnections; - private ScheduledFuture ackDeadlineUpdater; - private int streamAckDeadlineSeconds; - - public SubscriberImpl(SubscriberImpl.Builder builder) throws IOException { - receiver = builder.receiver; - maxOutstandingBytes = builder.maxOutstandingBytes; - maxOutstandingMessages = builder.maxOutstandingMessages; - subscription = builder.subscription; - ackExpirationPadding = builder.ackExpirationPadding; - streamAckDeadlineSeconds = - Math.max( - INITIAL_ACK_DEADLINE_SECONDS, - Ints.saturatedCast(ackExpirationPadding.getStandardSeconds())); - - flowController = - new FlowController(builder.maxOutstandingBytes, builder.maxOutstandingBytes, false); - - numChannels = Math.max(1, Runtime.getRuntime().availableProcessors()) * CHANNELS_PER_CORE; - executor = - builder.executor.isPresent() - ? builder.executor.get() - : Executors.newScheduledThreadPool( - numChannels * THREADS_PER_CHANNEL, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("cloud-pubsub-subscriber-thread-%d") - .build()); - - channelBuilder = - builder.channelBuilder.isPresent() - ? builder.channelBuilder.get() - : NettyChannelBuilder.forAddress(PUBSUB_API_ADDRESS, 443) - .maxMessageSize(MAX_INBOUND_MESSAGE_SIZE) - .flowControlWindow(5000000) // 2.5 MB - .negotiationType(NegotiationType.TLS) - .sslContext(GrpcSslContexts.forClient().ciphers(null).build()) - .executor(executor); - - credentials = - builder.credentials.isPresent() - ? builder.credentials.get() - : GoogleCredentials.getApplicationDefault() - .createScoped(Collections.singletonList(PUBSUB_API_SCOPE)); - - streamingSubscriberConnections = new ArrayList(numChannels); - pollingSubscriberConnections = new ArrayList(numChannels); - } - - @Override - protected void doStart() { - logger.debug("Starting subscriber group."); - startStreamingConnections(); - notifyStarted(); - } - - @Override - protected void doStop() { - stopAllStreamingConnections(); - stopAllPollingConnections(); - notifyStopped(); - } - - private void startStreamingConnections() { - synchronized (streamingSubscriberConnections) { - for (int i = 0; i < numChannels; i++) { - streamingSubscriberConnections.add( - new StreamingSubscriberConnection( - subscription, - credentials, - receiver, - ackExpirationPadding, - streamAckDeadlineSeconds, - ackLatencyDistribution, - channelBuilder.build(), - flowController, - executor)); - } - startConnections( - streamingSubscriberConnections, - new Listener() { - @Override - public void failed(State from, Throwable failure) { - // If a connection failed is because of a fatal error, we should fail the - // whole subscriber. - stopAllStreamingConnections(); - if (failure instanceof StatusRuntimeException - && ((StatusRuntimeException) failure).getStatus().getCode() - == Status.Code.UNIMPLEMENTED) { - logger.info("Unable to open streaming connections, falling back to polling."); - startPollingConnections(); - return; - } - notifyFailed(failure); - } - }); - } - - ackDeadlineUpdater = - executor.scheduleAtFixedRate( - new Runnable() { - @Override - public void run() { - // It is guaranteed this will be <= MAX_ACK_DEADLINE_SECONDS, the max of the API. - long ackLatency = - ackLatencyDistribution.getNthPercentile(PERCENTILE_FOR_ACK_DEADLINE_UPDATES); - if (ackLatency > 0) { - int possibleStreamAckDeadlineSeconds = - Math.max( - MIN_ACK_DEADLINE_SECONDS, - Ints.saturatedCast( - Math.max(ackLatency, ackExpirationPadding.getStandardSeconds()))); - if (streamAckDeadlineSeconds != possibleStreamAckDeadlineSeconds) { - streamAckDeadlineSeconds = possibleStreamAckDeadlineSeconds; - logger.debug( - "Updating stream deadline to {} seconds.", streamAckDeadlineSeconds); - for (StreamingSubscriberConnection subscriberConnection : - streamingSubscriberConnections) { - subscriberConnection.updateStreamAckDeadline(streamAckDeadlineSeconds); - } - } - } - } - }, - ACK_DEADLINE_UPDATE_PERIOD.getMillis(), - ACK_DEADLINE_UPDATE_PERIOD.getMillis(), - TimeUnit.MILLISECONDS); - } - - private void stopAllStreamingConnections() { - stopConnections(streamingSubscriberConnections); - ackDeadlineUpdater.cancel(true); - } - - private void startPollingConnections() { - synchronized (pollingSubscriberConnections) { - for (int i = 0; i < numChannels; i++) { - pollingSubscriberConnections.add( - new PollingSubscriberConnection( - subscription, - credentials, - receiver, - ackExpirationPadding, - ackLatencyDistribution, - channelBuilder.build(), - flowController, - executor)); - } - startConnections( - pollingSubscriberConnections, - new Listener() { - @Override - public void failed(State from, Throwable failure) { - // If a connection failed is because of a fatal error, we should fail the - // whole subscriber. - stopAllPollingConnections(); - try { - notifyFailed(failure); - } catch (IllegalStateException e) { - if (isRunning()) { - throw e; - } - // It could happen that we are shutting down while some channels fail. - } - } - }); - } - } - - private void stopAllPollingConnections() { - stopConnections(pollingSubscriberConnections); - } - - private void startConnections( - List connections, - final Listener connectionsListener) { - final CountDownLatch subscribersStarting = new CountDownLatch(numChannels); - for (final AbstractSubscriberConnection subscriber : connections) { - executor.submit( - new Runnable() { - @Override - public void run() { - subscriber.startAsync().awaitRunning(); - subscribersStarting.countDown(); - subscriber.addListener(connectionsListener, executor); - } - }); - } - try { - subscribersStarting.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - private void stopConnections(List connections) { - ArrayList liveConnections; - synchronized (connections) { - liveConnections = new ArrayList(connections); - connections.clear(); - } - final CountDownLatch connectionsStopping = new CountDownLatch(liveConnections.size()); - for (final AbstractSubscriberConnection subscriberConnection : liveConnections) { - executor.submit( - new Runnable() { - @Override - public void run() { - try { - subscriberConnection.stopAsync().awaitTerminated(); - } catch (IllegalStateException ignored) { - // It is expected for some connections to be already in state failed so stop will - // throw this expection. - } - connectionsStopping.countDown(); - } - }); - } - try { - connectionsStopping.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public SubscriberStats getStats() { - // TODO: Implement me - return null; - } - - @Override - public String getSubscription() { - return subscription; - } - - @Override - public Duration getAckExpirationPadding() { - return ackExpirationPadding; - } - - @Override - public Optional getMaxOutstandingMessages() { - return maxOutstandingMessages; - } - - @Override - public Optional getMaxOutstandingBytes() { - return maxOutstandingBytes; - } -} diff --git a/client/src/main/java/com/google/cloud/pubsub/SubscriberStats.java b/client/src/main/java/com/google/cloud/pubsub/SubscriberStats.java deleted file mode 100644 index 3d4637dc..00000000 --- a/client/src/main/java/com/google/cloud/pubsub/SubscriberStats.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.math.Stats; -import javax.annotation.concurrent.Immutable; - -/** - * A snapshot of the subscriber statistics at the time they were requested from the {@link - * Subscriber}. - */ -//TODO: Finish implementation. -@Immutable -public class SubscriberStats { - private final long totalReceivedMessages; - private final long totalAckedMessages; - private final Stats endToEndLatency; - private final Stats ackLatency; - private final long numberOfAutoExtendedAckDeadlines; - - SubscriberStats() { - this.totalReceivedMessages = 0; - this.totalAckedMessages = 0; - this.numberOfAutoExtendedAckDeadlines = 0; - this.endToEndLatency = null; - this.ackLatency = null; - } - - /** Number of successfully published messages. */ - public long getReceivedMessages() { - return totalReceivedMessages; - } - - /** Number of successfully published messages. */ - public long getAckedMessages() { - return totalAckedMessages; - } - - /** Number of received messages. */ - public long getTotalReceivedMessages() { - return totalReceivedMessages; - } - - /** Number messages acked. */ - public long getTotalAckedMessages() { - return totalAckedMessages; - } - - /** End to end latency. */ - public Stats getEndToEndLatency() { - return endToEndLatency; - } - - /** - * Acknowledgement latency; time in between the message has been received and then acknowledged or - * rejected. - */ - public Stats getAckLatency() { - return ackLatency; - } - - /** Number of messages for which we have auto extended its acknowledgement deadline. */ - public long getNumberOfAutoExtendedAckDeadlines() { - return numberOfAutoExtendedAckDeadlines; - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/FakeCredentials.java b/client/src/test/java/com/google/cloud/pubsub/FakeCredentials.java deleted file mode 100644 index d14c385f..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/FakeCredentials.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.auth.Credentials; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Map; - -/** - * Fake {@link Credentials}, meant to be used with an in-memory gRPC server. - */ -class FakeCredentials extends Credentials { - @Override - public String getAuthenticationType() { - return "None"; - } - - @Override - public Map> getRequestMetadata() throws IOException { - return null; - } - - @Override - public Map> getRequestMetadata(URI uri) throws IOException { - return null; - } - - @Override - public boolean hasRequestMetadata() { - return false; - } - - @Override - public boolean hasRequestMetadataOnly() { - return true; - } - - @Override - public void refresh() throws IOException { - // No-op - } -} \ No newline at end of file diff --git a/client/src/test/java/com/google/cloud/pubsub/FakePublisherServiceImpl.java b/client/src/test/java/com/google/cloud/pubsub/FakePublisherServiceImpl.java deleted file mode 100644 index 7657631b..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/FakePublisherServiceImpl.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.base.Optional; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PublisherGrpc.PublisherImplBase; -import io.grpc.stub.StreamObserver; -import java.util.Queue; -import java.util.concurrent.LinkedBlockingQueue; - -/** - * A fake implementation of {@link PublisherImplBase}, that can be used to test clients of a - * Cloud Pub/Sub Publisher. - */ -class FakePublisherServiceImpl extends PublisherImplBase { - - private final Queue publishResponses = new LinkedBlockingQueue<>(); - - /** - * Class used to save the state of a possible response. - */ - private static class Response { - Optional publishResponse; - Optional error; - - public Response(PublishResponse publishResponse) { - this.publishResponse = Optional.of(publishResponse); - this.error = Optional.absent(); - } - - public Response(Throwable exception) { - this.publishResponse = Optional.absent(); - this.error = Optional.of(exception); - } - - public PublishResponse getPublishResponse() { - return publishResponse.get(); - } - - public Throwable getError() { - return error.get(); - } - - boolean isError() { - return error.isPresent(); - } - } - - @Override - public void publish(PublishRequest request, StreamObserver responseObserver) { - Response response = null; - synchronized (publishResponses) { - response = publishResponses.poll(); - try { - if (response.isError()) { - responseObserver.onError(response.getError()); - return; - } - - responseObserver.onNext(response.getPublishResponse()); - responseObserver.onCompleted(); - } finally { - publishResponses.notifyAll(); - } - } - } - - public FakePublisherServiceImpl addPublishResponse(PublishResponse publishResponse) { - synchronized (publishResponses) { - publishResponses.add(new Response(publishResponse)); - } - return this; - } - - public FakePublisherServiceImpl addPublishResponse( - PublishResponse.Builder publishResponseBuilder) { - addPublishResponse(publishResponseBuilder.build()); - return this; - } - - public FakePublisherServiceImpl addPublishError(Throwable error) { - synchronized (publishResponses) { - publishResponses.add(new Response(error)); - } - return this; - } - - public void reset() { - synchronized (publishResponses) { - publishResponses.clear(); - publishResponses.notifyAll(); - } - } - - public void waitForNoOutstandingResponses() throws InterruptedException { - synchronized (publishResponses) { - while (!publishResponses.isEmpty()) { - publishResponses.wait(); - } - } - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/FakeScheduledExecutorService.java b/client/src/test/java/com/google/cloud/pubsub/FakeScheduledExecutorService.java deleted file mode 100644 index 0a0b843c..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/FakeScheduledExecutorService.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.common.primitives.Ints; -import com.google.common.util.concurrent.SettableFuture; -import java.util.ArrayList; -import java.util.List; -import java.util.PriorityQueue; -import java.util.concurrent.AbstractExecutorService; -import java.util.concurrent.Callable; -import java.util.concurrent.Delayed; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import org.joda.time.DateTime; -import org.joda.time.DateTimeUtils; -import org.joda.time.Duration; -import org.joda.time.MutableDateTime; - -/** - * Fake implementation of {@link ScheduledExecutorService} that allows tests control the reference - * time of the executor and decide when to execute any outstanding task. - */ -public class FakeScheduledExecutorService extends AbstractExecutorService - implements ScheduledExecutorService { - - private final AtomicBoolean shutdown = new AtomicBoolean(false); - private final PriorityQueue> pendingCallables = new PriorityQueue<>(); - private final MutableDateTime currentTime = MutableDateTime.now(); - - public FakeScheduledExecutorService() { - DateTimeUtils.setCurrentMillisFixed(currentTime.getMillis()); - } - - @Override - public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { - return schedulePendingCallable( - new PendingCallable<>( - new Duration(unit.toMillis(delay)), command, PendingCallableType.NORMAL)); - } - - @Override - public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { - return schedulePendingCallable( - new PendingCallable<>( - new Duration(unit.toMillis(delay)), callable, PendingCallableType.NORMAL)); - } - - @Override - public ScheduledFuture scheduleAtFixedRate( - Runnable command, long initialDelay, long period, TimeUnit unit) { - return schedulePendingCallable( - new PendingCallable<>( - new Duration(unit.toMillis(initialDelay)), command, PendingCallableType.FIXED_RATE)); - } - - @Override - public ScheduledFuture scheduleWithFixedDelay( - Runnable command, long initialDelay, long delay, TimeUnit unit) { - return schedulePendingCallable( - new PendingCallable<>( - new Duration(unit.toMillis(initialDelay)), command, PendingCallableType.FIXED_DELAY)); - } - - public void tick(long time, TimeUnit unit) { - advanceTime(Duration.millis(unit.toMillis(time))); - } - - /** - * This will advance the reference time of the executor and execute (in the same thread) any - * outstanding callable which execution time has passed. - */ - public void advanceTime(Duration toAdvance) { - currentTime.add(toAdvance); - DateTimeUtils.setCurrentMillisFixed(currentTime.getMillis()); - synchronized (pendingCallables) { - while (!pendingCallables.isEmpty() - && pendingCallables.peek().getScheduledTime().compareTo(currentTime) <= 0) { - try { - pendingCallables.poll().call(); - if (shutdown.get() && pendingCallables.isEmpty()) { - pendingCallables.notifyAll(); - } - } catch (Exception e) { - // We ignore any callable exception, which should be set to the future but not relevant to - // advanceTime. - } - } - } - } - - @Override - public void shutdown() { - if (shutdown.getAndSet(true)) { - throw new IllegalStateException("This executor has been shutdown already"); - } - } - - @Override - public List shutdownNow() { - if (shutdown.getAndSet(true)) { - throw new IllegalStateException("This executor has been shutdown already"); - } - List pending = new ArrayList<>(); - for (final PendingCallable pendingCallable : pendingCallables) { - pending.add( - new Runnable() { - @Override - public void run() { - pendingCallable.call(); - } - }); - } - synchronized (pendingCallables) { - pendingCallables.notifyAll(); - pendingCallables.clear(); - } - return pending; - } - - @Override - public boolean isShutdown() { - return shutdown.get(); - } - - @Override - public boolean isTerminated() { - return pendingCallables.isEmpty(); - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - synchronized (pendingCallables) { - if (pendingCallables.isEmpty()) { - return true; - } - pendingCallables.wait(unit.toMillis(timeout)); - return pendingCallables.isEmpty(); - } - } - - @Override - public void execute(Runnable command) { - if (shutdown.get()) { - throw new IllegalStateException("This executor has been shutdown"); - } - command.run(); - } - - ScheduledFuture schedulePendingCallable(PendingCallable callable) { - if (shutdown.get()) { - throw new IllegalStateException("This executor has been shutdown"); - } - synchronized (pendingCallables) { - pendingCallables.add(callable); - } - return callable.getScheduledFuture(); - } - - static enum PendingCallableType { - NORMAL, - FIXED_RATE, - FIXED_DELAY - } - - /** Class that saves the state of an scheduled pending callable. */ - class PendingCallable implements Comparable> { - DateTime creationTime = currentTime.toDateTime(); - Duration delay; - Callable pendingCallable; - SettableFuture future = SettableFuture.create(); - AtomicBoolean cancelled = new AtomicBoolean(false); - AtomicBoolean done = new AtomicBoolean(false); - PendingCallableType type; - - PendingCallable(Duration delay, final Runnable runnable, PendingCallableType type) { - pendingCallable = - new Callable() { - @Override - public T call() throws Exception { - runnable.run(); - return null; - } - }; - this.type = type; - this.delay = delay; - } - - PendingCallable(Duration delay, Callable callable, PendingCallableType type) { - pendingCallable = callable; - this.type = type; - this.delay = delay; - } - - private DateTime getScheduledTime() { - return creationTime.plus(delay); - } - - ScheduledFuture getScheduledFuture() { - return new ScheduledFuture() { - @Override - public long getDelay(TimeUnit unit) { - return unit.convert( - new Duration(currentTime, getScheduledTime()).getMillis(), TimeUnit.MILLISECONDS); - } - - @Override - public int compareTo(Delayed o) { - return Ints.saturatedCast( - getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS)); - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - synchronized (this) { - cancelled.set(true); - return !done.get(); - } - } - - @Override - public boolean isCancelled() { - return cancelled.get(); - } - - @Override - public boolean isDone() { - return done.get(); - } - - @Override - public T get() throws InterruptedException, ExecutionException { - return future.get(); - } - - @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return future.get(timeout, unit); - } - }; - } - - T call() { - T result = null; - synchronized (this) { - if (cancelled.get()) { - return null; - } - try { - result = pendingCallable.call(); - future.set(result); - } catch (Exception e) { - future.setException(e); - } finally { - switch (type) { - case NORMAL: - done.set(true); - break; - case FIXED_DELAY: - this.creationTime = currentTime.toDateTime(); - schedulePendingCallable(this); - break; - case FIXED_RATE: - this.creationTime = this.creationTime.plus(delay); - schedulePendingCallable(this); - break; - default: - // Nothing to do - } - } - } - return result; - } - - @Override - public int compareTo(PendingCallable other) { - return getScheduledTime().compareTo(other.getScheduledTime()); - } - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/FakeSubscriberServiceImpl.java b/client/src/test/java/com/google/cloud/pubsub/FakeSubscriberServiceImpl.java deleted file mode 100644 index f705fe7e..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/FakeSubscriberServiceImpl.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import com.google.api.client.util.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.Empty; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.GetSubscriptionRequest; -import com.google.pubsub.v1.ModifyAckDeadlineRequest; -import com.google.pubsub.v1.PublisherGrpc.PublisherImplBase; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.StreamingPullRequest; -import com.google.pubsub.v1.StreamingPullResponse; -import com.google.pubsub.v1.SubscriberGrpc.SubscriberImplBase; -import com.google.pubsub.v1.Subscription; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.StatusException; -import io.grpc.stub.StreamObserver; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * A fake implementation of {@link PublisherImplBase}, that can be used to test clients of a Cloud - * Pub/Sub Publisher. - */ -class FakeSubscriberServiceImpl extends SubscriberImplBase { - private final AtomicBoolean subscriptionInitialized = new AtomicBoolean(false); - private String subscription = ""; - private final AtomicInteger messageAckDeadline = new AtomicInteger(); - private final List openedStreams = new ArrayList<>(); - private final List closedStreams = new ArrayList<>(); - private final List acks = new ArrayList<>(); - private final List modAckDeadlines = new ArrayList<>(); - private final List receivedPullRequest = new ArrayList<>(); - private final BlockingQueue pullResponses = new LinkedBlockingDeque<>(); - private int currentStream; - - public static enum CloseSide { - SERVER, - CLIENT - } - - public static final class ModifyAckDeadline { - private final String ackId; - private final long seconds; - - public ModifyAckDeadline(String ackId, long seconds) { - Preconditions.checkNotNull(ackId); - this.ackId = ackId; - this.seconds = seconds; - } - - public String getAckId() { - return ackId; - } - - public long getSeconds() { - return seconds; - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ModifyAckDeadline)) { - return false; - } - ModifyAckDeadline other = (ModifyAckDeadline) obj; - return other.ackId.equals(this.ackId) && other.seconds == this.seconds; - } - - @Override - public int hashCode() { - return ackId.hashCode(); - } - - @Override - public String toString() { - return "Ack ID: " + ackId + ", deadline seconds: " + seconds; - } - } - - private static class Stream { - private StreamObserver requestObserver; - private StreamObserver responseObserver; - } - - @Override - public StreamObserver streamingPull( - final StreamObserver responseObserver) { - final Stream stream = new Stream(); - stream.requestObserver = - new StreamObserver() { - @Override - public void onNext(StreamingPullRequest request) { - synchronized (stream) { - if (!request.getSubscription().isEmpty()) { - if (!subscription.isEmpty() && !subscription.equals(request.getSubscription())) { - responseObserver.onError( - new StatusException( - Status.fromCode(Code.ABORTED) - .withDescription("Can only set one subscription."))); - return; - } - - synchronized (subscriptionInitialized) { - if (subscription.isEmpty()) { - if (request.getStreamAckDeadlineSeconds() == 0) { - responseObserver.onError( - new StatusException( - Status.fromCode(Code.INVALID_ARGUMENT) - .withDescription( - "A stream must be initialized with a ack deadline."))); - } - - subscription = request.getSubscription(); - subscriptionInitialized.set(true); - subscriptionInitialized.notifyAll(); - } - } - addOpenedStream(stream); - stream.notifyAll(); - } - - if (request.getStreamAckDeadlineSeconds() > 0) { - synchronized (messageAckDeadline) { - messageAckDeadline.set(request.getStreamAckDeadlineSeconds()); - messageAckDeadline.notifyAll(); - } - } - if (subscription.isEmpty()) { - closeStream(stream); - responseObserver.onError( - new StatusException( - Status.fromCode(Code.ABORTED) - .withDescription( - "The stream has not been properly initialized with a " - + "subscription."))); - return; - } - if (request.getAckIdsCount() > 0) { - addReceivedAcks(request.getAckIdsList()); - } - if (request.getModifyDeadlineAckIdsCount() > 0) { - if (request.getModifyDeadlineAckIdsCount() - != request.getModifyDeadlineSecondsCount()) { - closeStream(stream); - responseObserver.onError( - new StatusException( - Status.fromCode(Code.ABORTED) - .withDescription("Invalid modify ack deadline request."))); - return; - } - Iterator ackIds = request.getModifyDeadlineAckIdsList().iterator(); - Iterator seconds = request.getModifyDeadlineSecondsList().iterator(); - while (ackIds.hasNext() && seconds.hasNext()) { - addReceivedModifyAckDeadline( - new ModifyAckDeadline(ackIds.next(), seconds.next())); - } - } - } - } - - @Override - public void onError(Throwable error) { - closeStream(stream); - } - - @Override - public void onCompleted() { - closeStream(stream); - stream.responseObserver.onCompleted(); - } - }; - stream.responseObserver = responseObserver; - - return stream.requestObserver; - } - - public void sendStreamingResponse(StreamingPullResponse pullResponse) - throws InterruptedException { - waitForRegistedSubscription(); - synchronized (openedStreams) { - openedStreams.get(getAndAdvanceCurrentStream()).responseObserver.onNext(pullResponse); - } - } - - public void setMessageAckDeadlineSeconds(int ackDeadline) { - messageAckDeadline.set(ackDeadline); - } - - public void enqueuePullResponse(PullResponse response) { - pullResponses.add(response); - } - - @Override - public void getSubscription( - GetSubscriptionRequest request, StreamObserver responseObserver) { - responseObserver.onNext( - Subscription.newBuilder() - .setName(request.getSubscription()) - .setAckDeadlineSeconds(messageAckDeadline.get()) - .setTopic("fake-topic") - .build()); - responseObserver.onCompleted(); - } - - @Override - public void pull(PullRequest request, StreamObserver responseObserver) { - receivedPullRequest.add(request); - try { - responseObserver.onNext(pullResponses.take()); - responseObserver.onCompleted(); - } catch (InterruptedException e) { - responseObserver.onError(e); - } - } - - @Override - public void acknowledge( - AcknowledgeRequest request, io.grpc.stub.StreamObserver responseObserver) { - addReceivedAcks(request.getAckIdsList()); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - - @Override - public void modifyAckDeadline( - ModifyAckDeadlineRequest request, StreamObserver responseObserver) { - for (String ackId : request.getAckIdsList()) { - addReceivedModifyAckDeadline(new ModifyAckDeadline(ackId, request.getAckDeadlineSeconds())); - } - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - - public void sendError(Throwable error) throws InterruptedException { - waitForRegistedSubscription(); - synchronized (openedStreams) { - Stream stream = openedStreams.get(getAndAdvanceCurrentStream()); - stream.responseObserver.onError(error); - closeStream(stream); - } - } - - public String waitForRegistedSubscription() throws InterruptedException { - synchronized (subscriptionInitialized) { - while (!subscriptionInitialized.get()) { - subscriptionInitialized.wait(); - } - } - return subscription; - } - - public List waitAndConsumeReceivedAcks(int expectedCount) throws InterruptedException { - synchronized (acks) { - while (acks.size() < expectedCount) { - acks.wait(); - } - List receivedAcksCopy = ImmutableList.copyOf(acks.subList(0, expectedCount)); - acks.removeAll(receivedAcksCopy); - return receivedAcksCopy; - } - } - - public List waitAndConsumeModifyAckDeadlines(int expectedCount) - throws InterruptedException { - synchronized (modAckDeadlines) { - while (modAckDeadlines.size() < expectedCount) { - modAckDeadlines.wait(); - } - List modAckDeadlinesCopy = - ImmutableList.copyOf(modAckDeadlines.subList(0, expectedCount)); - modAckDeadlines.removeAll(modAckDeadlinesCopy); - return modAckDeadlinesCopy; - } - } - - public int waitForClosedStreams(int expectedCount) throws InterruptedException { - synchronized (closedStreams) { - while (closedStreams.size() < expectedCount) { - closedStreams.wait(); - } - return closedStreams.size(); - } - } - - public int waitForOpenedStreams(int expectedCount) throws InterruptedException { - synchronized (openedStreams) { - while (openedStreams.size() < expectedCount) { - openedStreams.wait(); - } - return openedStreams.size(); - } - } - - public void waitForStreamAckDeadline(int expectedValue) throws InterruptedException { - synchronized (messageAckDeadline) { - while (messageAckDeadline.get() != expectedValue) { - messageAckDeadline.wait(); - } - } - } - - public int getOpenedStreamsCount() { - return openedStreams.size(); - } - - public int getClosedStreamsCount() { - return closedStreams.size(); - } - - public List getAcks() { - return acks; - } - - public List getModifyAckDeadlines() { - return modAckDeadlines; - } - - public void reset() { - synchronized (subscriptionInitialized) { - synchronized (openedStreams) { - synchronized (acks) { - synchronized (modAckDeadlines) { - openedStreams.clear(); - closedStreams.clear(); - acks.clear(); - modAckDeadlines.clear(); - subscriptionInitialized.set(false); - subscription = ""; - pullResponses.clear(); - receivedPullRequest.clear(); - currentStream = 0; - } - } - } - } - } - - private void addOpenedStream(Stream stream) { - synchronized (openedStreams) { - openedStreams.add(stream); - openedStreams.notifyAll(); - } - } - - private void closeStream(Stream stream) { - synchronized (openedStreams) { - openedStreams.remove(stream); - closedStreams.add(stream); - } - synchronized (closedStreams) { - closedStreams.notifyAll(); - } - } - - private int getAndAdvanceCurrentStream() { - int current = currentStream; - synchronized (openedStreams) { - currentStream = (currentStream + 1) % openedStreams.size(); - } - return current; - } - - private void addReceivedAcks(Collection newAckIds) { - synchronized (acks) { - acks.addAll(newAckIds); - acks.notifyAll(); - } - } - - private void addReceivedModifyAckDeadline(ModifyAckDeadline newAckDeadline) { - synchronized (modAckDeadlines) { - modAckDeadlines.add(newAckDeadline); - modAckDeadlines.notifyAll(); - } - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/FlowControllerTest.java b/client/src/test/java/com/google/cloud/pubsub/FlowControllerTest.java deleted file mode 100644 index 7fd9dfd8..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/FlowControllerTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import com.google.cloud.pubsub.Publisher.CloudPubsubFlowControlException; -import com.google.cloud.pubsub.Publisher.MaxOutstandingBytesReachedException; -import com.google.cloud.pubsub.Publisher.MaxOutstandingMessagesReachedException; -import com.google.common.base.Optional; -import com.google.common.util.concurrent.SettableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link FlowController}. */ -@RunWith(JUnit4.class) -public class FlowControllerTest { - - @Test - public void testReserveRelease_ok() throws Exception { - FlowController flowController = new FlowController(Optional.of(10), Optional.of(10), false); - - flowController.reserve(1, 1); - flowController.release(1, 1); - } - - @Test - public void testInvalidArguments() throws Exception { - FlowController flowController = new FlowController(Optional.of(10), Optional.of(10), false); - - flowController.reserve(1, 0); - try { - flowController.reserve(-1, 1); - fail("Must have thrown an illegal argument error"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - flowController.reserve(1, -1); - fail("Must have thrown an illegal argument error"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - flowController.reserve(0, 1); - fail("Must have thrown an illegal argument error"); - } catch (IllegalArgumentException expected) { - // Expected - } - } - - @Test - public void testReserveRelease_noLimits_ok() throws Exception { - FlowController flowController = - new FlowController(Optional.absent(), Optional.absent(), false); - - flowController.reserve(1, 1); - flowController.release(1, 1); - } - - @Test - public void testReserveRelease_blockedByNumberOfMessages() throws Exception { - FlowController flowController = new FlowController(Optional.of(10), Optional.of(100), false); - - testBlockingReserveRelease(flowController, 10, 10); - } - - @Test - public void testReserveRelease_blockedByNumberOfMessages_noBytesLimit() throws Exception { - FlowController flowController = - new FlowController(Optional.of(10), Optional.absent(), false); - - testBlockingReserveRelease(flowController, 10, 10); - } - - @Test - public void testReserveRelease_blockedByNumberOfBytes() throws Exception { - FlowController flowController = new FlowController(Optional.of(100), Optional.of(10), false); - - testBlockingReserveRelease(flowController, 10, 10); - } - - @Test - public void testReserveRelease_blockedByNumberOfBytes_noMessagesLimit() throws Exception { - FlowController flowController = - new FlowController(Optional.absent(), Optional.of(10), false); - - testBlockingReserveRelease(flowController, 10, 10); - } - - private static void testBlockingReserveRelease( - final FlowController flowController, final int maxNumMessages, final int maxNumBytes) - throws Exception { - - flowController.reserve(1, 1); - - final SettableFuture permitsReserved = SettableFuture.create(); - Future finished = - Executors.newCachedThreadPool() - .submit( - new Runnable() { - @Override - public void run() { - try { - permitsReserved.set(null); - flowController.reserve(maxNumMessages, maxNumBytes); - } catch (CloudPubsubFlowControlException e) { - throw new AssertionError(e); - } - } - }); - - permitsReserved.get(); - flowController.release(1, 1); - - finished.get(); - } - - @Test - public void testReserveRelease_rejectedByNumberOfMessages() throws Exception { - FlowController flowController = new FlowController(Optional.of(10), Optional.of(100), true); - - testRejectedReserveRelease( - flowController, 10, 10, MaxOutstandingMessagesReachedException.class); - } - - @Test - public void testReserveRelease_rejectedByNumberOfMessages_noBytesLimit() throws Exception { - FlowController flowController = - new FlowController(Optional.of(10), Optional.absent(), true); - - testRejectedReserveRelease( - flowController, 10, 10, MaxOutstandingMessagesReachedException.class); - } - - @Test - public void testReserveRelease_rejectedByNumberOfBytes() throws Exception { - FlowController flowController = new FlowController(Optional.of(100), Optional.of(10), true); - - testRejectedReserveRelease(flowController, 10, 10, MaxOutstandingBytesReachedException.class); - } - - @Test - public void testReserveRelease_rejectedByNumberOfBytes_noMessagesLimit() throws Exception { - FlowController flowController = - new FlowController(Optional.absent(), Optional.of(10), true); - - testRejectedReserveRelease(flowController, 10, 10, MaxOutstandingBytesReachedException.class); - } - - private void testRejectedReserveRelease( - FlowController flowController, - int maxNumMessages, - int maxNumBytes, - Class expectedException) - throws CloudPubsubFlowControlException { - - flowController.reserve(1, 1); - - try { - flowController.reserve(maxNumMessages, maxNumBytes); - fail("Should thrown a CloudPubsubFlowControlException"); - } catch (CloudPubsubFlowControlException e) { - assertTrue(expectedException.isInstance(e)); - } - - flowController.release(1, 1); - - flowController.reserve(maxNumMessages, maxNumBytes); - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/MessagesWaiterTest.java b/client/src/test/java/com/google/cloud/pubsub/MessagesWaiterTest.java deleted file mode 100644 index 753bd10f..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/MessagesWaiterTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import static org.junit.Assert.assertEquals; - -import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Tests for {@link PublisherImpl}. - */ -@RunWith(JUnit4.class) -public class MessagesWaiterTest { - - @Test - public void test() throws Exception { - final MessagesWaiter waiter = new MessagesWaiter(); - waiter.incrementPendingMessages(1); - - final AtomicBoolean waitReached = new AtomicBoolean(); - - Thread t = new Thread(new Runnable(){ - @Override - public void run() { - while (!waitReached.get()) { - Thread.yield(); - } - waiter.incrementPendingMessages(-1); - } - }); - t.start(); - - waiter.waitNoMessages(waitReached); - t.join(); - - assertEquals(0, waiter.pendingMessages()); - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/PublisherImplTest.java b/client/src/test/java/com/google/cloud/pubsub/PublisherImplTest.java deleted file mode 100644 index fece0268..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/PublisherImplTest.java +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.times; - -import com.google.cloud.pubsub.Publisher.Builder; -import com.google.common.base.Optional; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PubsubMessage; -import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.ServerImpl; -import io.grpc.stub.StreamObserver; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import org.joda.time.Duration; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -/** Tests for {@link PublisherImpl}. */ -@RunWith(JUnit4.class) -public class PublisherImplTest { - - private static final String TEST_TOPIC = "projects/test-project/topics/test-topic"; - - private static InProcessChannelBuilder testChannelBuilder; - - @Captor private ArgumentCaptor requestCaptor; - - private FakeScheduledExecutorService fakeExecutor; - - private FakeCredentials testCredentials; - - private static FakePublisherServiceImpl testPublisherServiceImpl; - - private static ServerImpl testServer; - - @BeforeClass - public static void setUpClass() throws Exception { - testPublisherServiceImpl = Mockito.spy(new FakePublisherServiceImpl()); - - InProcessServerBuilder serverBuilder = InProcessServerBuilder.forName("test-server"); - testChannelBuilder = InProcessChannelBuilder.forName("test-server"); - InProcessChannelBuilder.forName("publisher_test"); - serverBuilder.addService(testPublisherServiceImpl); - testServer = serverBuilder.build(); - testServer.start(); - } - - @Before - public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); - testPublisherServiceImpl.reset(); - Mockito.reset(testPublisherServiceImpl); - fakeExecutor = new FakeScheduledExecutorService(); - testCredentials = new FakeCredentials(); - } - - @AfterClass - public static void tearDownClass() throws Exception { - testServer.shutdownNow().awaitTermination(); - } - - @Test - public void testPublishByDuration() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setMaxBatchDuration(Duration.standardSeconds(5)) - // To demonstrate that reaching duration will trigger publish - .setMaxBatchMessages(10) - .build(); - - testPublisherServiceImpl.addPublishResponse( - PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")); - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - ListenableFuture publishFuture2 = sendTestMessage(publisher, "B"); - - assertFalse(publishFuture1.isDone()); - assertFalse(publishFuture2.isDone()); - - fakeExecutor.advanceTime(Duration.standardSeconds(10)); - - assertEquals("1", publishFuture1.get()); - assertEquals("2", publishFuture2.get()); - - Mockito.verify(testPublisherServiceImpl) - .publish(requestCaptor.capture(), Mockito.>any()); - assertEquals(2, requestCaptor.getValue().getMessagesCount()); - } - - @Test - public void testPublishByNumBatchedMessages() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setMaxBatchDuration(Duration.standardSeconds(100)) - .setMaxBatchMessages(2) - .build(); - - testPublisherServiceImpl - .addPublishResponse(PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")) - .addPublishResponse(PublishResponse.newBuilder().addMessageIds("3").addMessageIds("4")); - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - ListenableFuture publishFuture2 = sendTestMessage(publisher, "B"); - ListenableFuture publishFuture3 = sendTestMessage(publisher, "C"); - - // Note we are not advancing time but message should still get published - - assertEquals("1", publishFuture1.get()); - assertEquals("2", publishFuture2.get()); - - assertFalse(publishFuture3.isDone()); - - ListenableFuture publishFuture4 = - publisher.publish(PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("D")).build()); - - assertEquals("3", publishFuture3.get()); - assertEquals("4", publishFuture4.get()); - - Mockito.verify(testPublisherServiceImpl, times(2)) - .publish(requestCaptor.capture(), Mockito.>any()); - assertEquals(2, requestCaptor.getAllValues().get(0).getMessagesCount()); - assertEquals(2, requestCaptor.getAllValues().get(1).getMessagesCount()); - } - - @Test - public void testSinglePublishByNumBytes() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setMaxBatchDuration(Duration.standardSeconds(100)) - .setMaxBatchMessages(2) - .build(); - - testPublisherServiceImpl - .addPublishResponse(PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")) - .addPublishResponse(PublishResponse.newBuilder().addMessageIds("3").addMessageIds("4")); - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - ListenableFuture publishFuture2 = sendTestMessage(publisher, "B"); - ListenableFuture publishFuture3 = sendTestMessage(publisher, "C"); - - // Note we are not advancing time but message should still get published - - assertEquals("1", publishFuture1.get()); - assertEquals("2", publishFuture2.get()); - assertFalse(publishFuture3.isDone()); - - ListenableFuture publishFuture4 = sendTestMessage(publisher, "D"); - assertEquals("3", publishFuture3.get()); - assertEquals("4", publishFuture4.get()); - - Mockito.verify(testPublisherServiceImpl, times(2)) - .publish(requestCaptor.capture(), Mockito.>any()); - } - - @Test - public void testPublishMixedSizeAndDuration() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setMaxBatchDuration(Duration.standardSeconds(5)) - // To demonstrate that reaching duration will trigger publish - .setMaxBatchMessages(2) - .build(); - - testPublisherServiceImpl.addPublishResponse( - PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")); - testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("3")); - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - - fakeExecutor.advanceTime(Duration.standardSeconds(2)); - assertFalse(publishFuture1.isDone()); - - ListenableFuture publishFuture2 = sendTestMessage(publisher, "B"); - - // Publishing triggered by batch size - assertEquals("1", publishFuture1.get()); - assertEquals("2", publishFuture2.get()); - - ListenableFuture publishFuture3 = sendTestMessage(publisher, "C"); - - assertFalse(publishFuture3.isDone()); - - // Publishing triggered by time - fakeExecutor.advanceTime(Duration.standardSeconds(5)); - - assertEquals("3", publishFuture3.get()); - - Mockito.verify(testPublisherServiceImpl, times(2)) - .publish(requestCaptor.capture(), Mockito.>any()); - assertEquals(2, requestCaptor.getAllValues().get(0).getMessagesCount()); - assertEquals(1, requestCaptor.getAllValues().get(1).getMessagesCount()); - } - - private ListenableFuture sendTestMessage(Publisher publisher, String data) { - return publisher.publish( - PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(data)).build()); - } - - @Test - public void testPublishFailureRetries() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setExecutor(Executors.newSingleThreadScheduledExecutor()) - .setMaxBatchDuration(Duration.standardSeconds(5)) - .setMaxBatchMessages(1) - .build(); // To demonstrate that reaching duration will trigger publish - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - - testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); - testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); - - assertEquals("1", publishFuture1.get()); - - Mockito.verify(testPublisherServiceImpl, times(2)) - .publish(Mockito.any(), Mockito.>any()); - } - - @Test(expected = Throwable.class) - public void testPublishFailureRetries_exceededsRetryDuration() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setExecutor(Executors.newSingleThreadScheduledExecutor()) - .setSendBatchDeadline(Duration.standardSeconds(10)) - .setMaxBatchDuration(Duration.standardSeconds(5)) - .setMaxBatchMessages(1) - .build(); // To demonstrate that reaching duration will trigger publish - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - - // With exponential backoff, starting at 5ms we should have no more than 11 retries - for (int i = 0; i < 11; ++i) { - testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); - } - - try { - publishFuture1.get(); - } finally { - Mockito.verify(testPublisherServiceImpl, atLeast(10)) - .publish(Mockito.any(), Mockito.>any()); - } - } - - @Test(expected = ExecutionException.class) - public void testPublishFailureRetries_nonRetryableFailsImmediately() throws Exception { - Publisher publisher = - getTestPublisherBuilder() - .setExecutor(Executors.newSingleThreadScheduledExecutor()) - .setSendBatchDeadline(Duration.standardSeconds(10)) - .setMaxBatchDuration(Duration.standardSeconds(5)) - .setMaxBatchMessages(1) - .build(); // To demonstrate that reaching duration will trigger publish - - ListenableFuture publishFuture1 = sendTestMessage(publisher, "A"); - - testPublisherServiceImpl.addPublishError(new StatusException(Status.INVALID_ARGUMENT)); - - try { - publishFuture1.get(); - } finally { - Mockito.verify(testPublisherServiceImpl) - .publish(Mockito.any(), Mockito.>any()); - } - } - - @Test - public void testPublisherGetters() throws Exception { - FakeCredentials credentials = new FakeCredentials(); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - - Publisher.Builder builder = Publisher.Builder.newBuilder(TEST_TOPIC); - builder.setChannelBuilder(testChannelBuilder); - builder.setCredentials(credentials); - builder.setExecutor(executor); - builder.setFailOnFlowControlLimits(true); - builder.setMaxBatchBytes(10); - builder.setMaxBatchDuration(new Duration(11)); - builder.setMaxBatchMessages(12); - builder.setMaxOutstandingBytes(13); - builder.setMaxOutstandingMessages(14); - builder.setRequestTimeout(new Duration(15)); - builder.setSendBatchDeadline(new Duration(16000)); - Publisher publisher = builder.build(); - - assertEquals(TEST_TOPIC, publisher.getTopic()); - assertEquals(10, publisher.getMaxBatchBytes()); - assertEquals(new Duration(11), publisher.getMaxBatchDuration()); - assertEquals(12, publisher.getMaxBatchMessages()); - assertEquals(Optional.of(13), publisher.getMaxOutstandingBytes()); - assertEquals(Optional.of(14), publisher.getMaxOutstandingMessages()); - assertTrue(publisher.failOnFlowControlLimits()); - } - - @Test - public void testBuilderParametersAndDefaults() { - Publisher.Builder builder = Publisher.Builder.newBuilder(TEST_TOPIC); - assertEquals(TEST_TOPIC, builder.topic); - assertEquals(Optional.absent(), builder.channelBuilder); - assertEquals(Optional.absent(), builder.executor); - assertFalse(builder.failOnFlowControlLimits); - assertEquals(Publisher.DEFAULT_MAX_BATCH_BYTES, builder.maxBatchBytes); - assertEquals(Publisher.DEFAULT_MAX_BATCH_DURATION, builder.maxBatchDuration); - assertEquals(Publisher.DEFAULT_MAX_BATCH_MESSAGES, builder.maxBatchMessages); - assertEquals(Optional.absent(), builder.maxOutstandingBytes); - assertEquals(Optional.absent(), builder.maxOutstandingMessages); - assertEquals(Publisher.DEFAULT_REQUEST_TIMEOUT, builder.requestTimeout); - assertEquals(Publisher.MIN_SEND_BATCH_DURATION, builder.sendBatchDeadline); - assertEquals(Optional.absent(), builder.userCredentials); - } - - @Test - public void testBuilderInvalidArguments() { - Publisher.Builder builder = Publisher.Builder.newBuilder(TEST_TOPIC); - - try { - builder.setChannelBuilder(null); - fail("Should have thrown an IllegalArgumentException"); - } catch (NullPointerException expected) { - // Expected - } - - try { - builder.setCredentials(null); - fail("Should have thrown an IllegalArgumentException"); - } catch (NullPointerException expected) { - // Expected - } - - try { - builder.setExecutor(null); - fail("Should have thrown an IllegalArgumentException"); - } catch (NullPointerException expected) { - // Expected - } - try { - builder.setMaxBatchBytes(0); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - builder.setMaxBatchBytes(-1); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - - builder.setMaxBatchDuration(new Duration(1)); - try { - builder.setMaxBatchDuration(null); - fail("Should have thrown an IllegalArgumentException"); - } catch (NullPointerException expected) { - // Expected - } - try { - builder.setMaxBatchDuration(new Duration(-1)); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - - builder.setMaxBatchMessages(1); - try { - builder.setMaxBatchMessages(0); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - builder.setMaxBatchMessages(-1); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - - builder.setMaxOutstandingBytes(1); - try { - builder.setMaxOutstandingBytes(0); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - builder.setMaxOutstandingBytes(-1); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - - builder.setMaxOutstandingMessages(1); - try { - builder.setMaxOutstandingMessages(0); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - try { - builder.setMaxOutstandingMessages(-1); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - - builder.setRequestTimeout(Publisher.MIN_REQUEST_TIMEOUT); - try { - builder.setRequestTimeout(Publisher.MIN_REQUEST_TIMEOUT.minus(1)); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - builder.setSendBatchDeadline(Publisher.MIN_SEND_BATCH_DURATION); - try { - builder.setSendBatchDeadline(Publisher.MIN_SEND_BATCH_DURATION.minus(1)); - fail("Should have thrown an IllegalArgumentException"); - } catch (IllegalArgumentException expected) { - // Expected - } - } - - private Builder getTestPublisherBuilder() { - return Publisher.Builder.newBuilder(TEST_TOPIC) - .setCredentials(testCredentials) - .setExecutor(fakeExecutor) - .setChannelBuilder(testChannelBuilder); - } -} diff --git a/client/src/test/java/com/google/cloud/pubsub/SubscriberImplTest.java b/client/src/test/java/com/google/cloud/pubsub/SubscriberImplTest.java deleted file mode 100644 index d96be6c5..00000000 --- a/client/src/test/java/com/google/cloud/pubsub/SubscriberImplTest.java +++ /dev/null @@ -1,498 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.pubsub; - -import static org.junit.Assert.assertEquals; - -import com.google.cloud.pubsub.FakeSubscriberServiceImpl.ModifyAckDeadline; -import com.google.cloud.pubsub.Subscriber.Builder; -import com.google.cloud.pubsub.Subscriber.MessageReceiver; -import com.google.cloud.pubsub.Subscriber.MessageReceiver.AckReply; -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.Service.State; -import com.google.common.util.concurrent.SettableFuture; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import com.google.pubsub.v1.StreamingPullResponse; -import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.StatusRuntimeException; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.ServerImpl; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Deque; -import java.util.List; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import org.joda.time.Duration; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -/** Tests for {@link SubscriberImpl}. */ -@RunWith(Parameterized.class) -public class SubscriberImplTest { - - private static final String TEST_SUBSCRIPTION = - "projects/test-project/subscriptions/test-subscription"; - - private static final PubsubMessage TEST_MESSAGE = - PubsubMessage.newBuilder().setMessageId("1").build(); - - @Parameters - public static Collection data() { - return Arrays.asList(new Object[][] {{true}, {false}}); - } - - private final boolean isStreamingTest; - - private InProcessChannelBuilder testChannelBuilder; - private FakeScheduledExecutorService fakeExecutor; - private FakeSubscriberServiceImpl fakeSubscriberServiceImpl; - private ServerImpl testServer; - - private FakeCredentials testCredentials; - private TestReceiver testReceiver; - - static class TestReceiver implements MessageReceiver { - private final Deque> outstandingMessageReplies = - new ConcurrentLinkedDeque<>(); - private AckReply ackReply = AckReply.ACK; - private Optional messageCountLatch = Optional.absent(); - private Optional error = Optional.absent(); - private boolean explicitAckReplies; - - void setReply(AckReply ackReply) { - this.ackReply = ackReply; - } - - void setErrorReply(Throwable error) { - this.error = Optional.of(error); - } - - void setExplicitAck(boolean explicitAckReplies) { - this.explicitAckReplies = explicitAckReplies; - } - - void setExpectedMessages(int expected) { - this.messageCountLatch = Optional.of(new CountDownLatch(expected)); - } - - void waitForExpectedMessages() throws InterruptedException { - if (messageCountLatch.isPresent()) { - messageCountLatch.get().await(); - } - } - - @Override - public ListenableFuture receiveMessage(PubsubMessage message) { - if (messageCountLatch.isPresent()) { - messageCountLatch.get().countDown(); - } - SettableFuture reply = SettableFuture.create(); - - if (explicitAckReplies) { - outstandingMessageReplies.add(reply); - } else { - if (error.isPresent()) { - reply.setException(error.get()); - } else { - reply.set(ackReply); - } - } - - return reply; - } - - public void replyNextOutstandingMessage() { - Preconditions.checkState(explicitAckReplies); - - SettableFuture reply = outstandingMessageReplies.poll(); - if (error.isPresent()) { - reply.setException(error.get()); - } else { - reply.set(ackReply); - } - } - - public void replyAllOutstandingMessage() { - Preconditions.checkState(explicitAckReplies); - - while (!outstandingMessageReplies.isEmpty()) { - replyNextOutstandingMessage(); - } - } - } - - public SubscriberImplTest(boolean streamingTest) { - this.isStreamingTest = streamingTest; - } - - @Rule public TestName testName = new TestName(); - - @Before - public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); - - InProcessServerBuilder serverBuilder = InProcessServerBuilder.forName(testName.getMethodName()); - fakeSubscriberServiceImpl = Mockito.spy(new FakeSubscriberServiceImpl()); - fakeExecutor = new FakeScheduledExecutorService(); - testChannelBuilder = InProcessChannelBuilder.forName(testName.getMethodName()); - serverBuilder.addService(fakeSubscriberServiceImpl); - testServer = serverBuilder.build(); - testServer.start(); - - testReceiver = new TestReceiver(); - testCredentials = new FakeCredentials(); - } - - @After - public void tearDown() throws Exception { - testServer.shutdownNow().awaitTermination(); - fakeSubscriberServiceImpl.reset(); - } - - @Test - public void testAckSingleMessage() throws Exception { - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - List testAckIds = ImmutableList.of("A"); - sendMessages(testAckIds); - - // Trigger ack sending - subscriber.stopAsync().awaitTerminated(); - - assertEquivalent(testAckIds, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(1)); - } - - @Test - public void testNackSingleMessage() throws Exception { - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - testReceiver.setReply(AckReply.NACK); - sendMessages(ImmutableList.of("A")); - - // Trigger ack sending - subscriber.stopAsync().awaitTerminated(); - - assertEquivalent( - ImmutableList.of(new ModifyAckDeadline("A", 0)), - fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(1)); - } - - @Test - public void testReceiverError_NacksMessage() throws Exception { - testReceiver.setErrorReply(new Exception("Can't process message")); - - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - sendMessages(ImmutableList.of("A")); - - // Trigger nack sending - subscriber.stopAsync().awaitTerminated(); - // TODO: it's really unstable test and cause of most travis failures. To solve it I decided to use sleep at the time. But we should find a real reason of it or modify requirements. - Thread.sleep(200L); - - assertEquivalent( - ImmutableList.of(new ModifyAckDeadline("A", 0)), - fakeSubscriberServiceImpl.getModifyAckDeadlines()); - } - - @Test - public void testBatchAcks() throws Exception { - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - List testAckIdsBatch1 = ImmutableList.of("A", "B", "C"); - sendMessages(testAckIdsBatch1); - - // Trigger ack sending - fakeExecutor.advanceTime(StreamingSubscriberConnection.PENDING_ACKS_SEND_DELAY); - - assertEquivalent(testAckIdsBatch1, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(3)); - - // Ensures the next ack sending alarm gets properly setup - List testAckIdsBatch2 = ImmutableList.of("D", "E"); - sendMessages(testAckIdsBatch2); - - fakeExecutor.advanceTime(StreamingSubscriberConnection.PENDING_ACKS_SEND_DELAY); - - assertEquivalent(testAckIdsBatch2, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(2)); - - subscriber.stopAsync().awaitTerminated(); - } - - @Test - public void testBatchAcksAndNacks() throws Exception { - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - // Send messages to be acked - List testAckIdsBatch1 = ImmutableList.of("A", "B", "C"); - sendMessages(testAckIdsBatch1); - - // Send messages to be nacked - List testAckIdsBatch2 = ImmutableList.of("D", "E"); - // Nack messages - testReceiver.setReply(AckReply.NACK); - sendMessages(testAckIdsBatch2); - - // Trigger ack sending - fakeExecutor.advanceTime(StreamingSubscriberConnection.PENDING_ACKS_SEND_DELAY); - - assertEquivalent(testAckIdsBatch1, fakeSubscriberServiceImpl.waitAndConsumeReceivedAcks(3)); - assertEquivalent( - ImmutableList.of(new ModifyAckDeadline("D", 0), new ModifyAckDeadline("E", 0)), - fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(2)); - - subscriber.stopAsync().awaitTerminated(); - } - - @Test - public void testModifyAckDeadline() throws Exception { - Subscriber subscriber = - startSubscriber( - getTestSubscriberBuilder(testReceiver) - .setAckExpirationPadding(Duration.standardSeconds(1))); - - // Send messages to be acked - List testAckIdsBatch = ImmutableList.of("A", "B", "C"); - testReceiver.setExplicitAck(true); - sendMessages(testAckIdsBatch); - - // Trigger modify ack deadline sending - 10s initial stream ack deadline - 1 padding - fakeExecutor.advanceTime(Duration.standardSeconds(9)); - - assertEquivalentWithTransformation( - testAckIdsBatch, - fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), - new Function() { - @Override - public ModifyAckDeadline apply(String ack) { - return new ModifyAckDeadline(ack, 2); // 2 seconds is the initial mod ack deadline - } - }); - - // Trigger modify ack deadline sending - 2s of the renewed deadlines - fakeExecutor.advanceTime(Duration.standardSeconds(2)); - - assertEquivalentWithTransformation( - testAckIdsBatch, - fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), - new Function() { - @Override - public ModifyAckDeadline apply(String ack) { - return new ModifyAckDeadline(ack, 4); - } - }); - - testReceiver.replyAllOutstandingMessage(); - subscriber.stopAsync().awaitTerminated(); - } - - @Test - public void testStreamAckDeadlineUpdate() throws Exception { - if (!isStreamingTest) { - // This test is not applicable to polling. - return; - } - - Subscriber subscriber = - startSubscriber( - getTestSubscriberBuilder(testReceiver) - .setAckExpirationPadding(Duration.standardSeconds(1))); - - fakeSubscriberServiceImpl.waitForStreamAckDeadline(10); - - // Send messages to be acked - testReceiver.setExplicitAck(true); - sendMessages(ImmutableList.of("A")); - - // Make the ack latency of the receiver equals 20 seconds - fakeExecutor.advanceTime(Duration.standardSeconds(20)); - testReceiver.replyNextOutstandingMessage(); - - // Wait for an ack deadline update - fakeExecutor.advanceTime(Duration.standardSeconds(60)); - - fakeSubscriberServiceImpl.waitForStreamAckDeadline(20); - - // Send more messages to be acked - testReceiver.setExplicitAck(true); - for (int i = 0; i < 999; i++) { - sendMessages(ImmutableList.of(Integer.toString(i))); - } - - // Reduce the 99th% ack latency of the receiver to 10 seconds - fakeExecutor.advanceTime(Duration.standardSeconds(10)); - for (int i = 0; i < 999; i++) { - testReceiver.replyNextOutstandingMessage(); - } - - // Wait for an ack deadline update - fakeExecutor.advanceTime(Duration.standardSeconds(60)); - - fakeSubscriberServiceImpl.waitForStreamAckDeadline(10); - - subscriber.stopAsync().awaitTerminated(); - } - - @Test - public void testOpenedChannels() throws Exception { - if (!isStreamingTest) { - // This test is not applicable to polling. - return; - } - - final int expectedChannelCount = - Runtime.getRuntime().availableProcessors() * SubscriberImpl.CHANNELS_PER_CORE; - - Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver)); - - assertEquals( - expectedChannelCount, fakeSubscriberServiceImpl.waitForOpenedStreams(expectedChannelCount)); - - subscriber.stopAsync().awaitTerminated(); - } - - @Test - public void testFailedChannel_recoverableError_channelReopened() throws Exception { - if (!isStreamingTest) { - // This test is not applicable to polling. - return; - } - - final int expectedChannelCount = - Runtime.getRuntime().availableProcessors() * SubscriberImpl.CHANNELS_PER_CORE; - - Subscriber subscriber = - startSubscriber( - getTestSubscriberBuilder(testReceiver) - .setExecutor(Executors.newSingleThreadScheduledExecutor())); - - // Recoverable error - fakeSubscriberServiceImpl.sendError(new StatusException(Status.INTERNAL)); - - assertEquals(1, fakeSubscriberServiceImpl.waitForClosedStreams(1)); - assertEquals( - expectedChannelCount, fakeSubscriberServiceImpl.waitForOpenedStreams(expectedChannelCount)); - - subscriber.stopAsync().awaitTerminated(); - } - - @Test(expected = IllegalStateException.class) - public void testFailedChannel_fatalError_subscriberFails() throws Exception { - if (!isStreamingTest) { - // This test is not applicable to polling. - throw new IllegalStateException("To fullfil test expectation"); - } - - Subscriber subscriber = - startSubscriber( - getTestSubscriberBuilder(testReceiver) - .setExecutor(Executors.newScheduledThreadPool(10))); - - // Fatal error - fakeSubscriberServiceImpl.sendError(new StatusException(Status.INVALID_ARGUMENT)); - - try { - subscriber.awaitTerminated(); - } finally { - // The subscriber must finish with an state error because its FAILED status. - assertEquals(State.FAILED, subscriber.state()); - assertEquals( - Status.INVALID_ARGUMENT, - ((StatusRuntimeException) subscriber.failureCause()).getStatus()); - } - } - - private Subscriber startSubscriber(Builder testSubscriberBuilder) throws Exception { - Subscriber subscriber = testSubscriberBuilder.build(); - subscriber.startAsync().awaitRunning(); - if (!isStreamingTest) { - // Shutdown streaming - fakeSubscriberServiceImpl.sendError(new StatusException(Status.UNIMPLEMENTED)); - } - return subscriber; - } - - private void sendMessages(Iterable ackIds) throws InterruptedException { - List messages = new ArrayList(); - for (String ackId : ackIds) { - messages.add(ReceivedMessage.newBuilder().setAckId(ackId).setMessage(TEST_MESSAGE).build()); - } - testReceiver.setExpectedMessages(messages.size()); - if (isStreamingTest) { - fakeSubscriberServiceImpl.sendStreamingResponse( - StreamingPullResponse.newBuilder().addAllReceivedMessages(messages).build()); - } else { - fakeSubscriberServiceImpl.enqueuePullResponse( - PullResponse.newBuilder().addAllReceivedMessages(messages).build()); - } - testReceiver.waitForExpectedMessages(); - } - - private Builder getTestSubscriberBuilder(MessageReceiver receiver) { - return Subscriber.Builder.newBuilder(TEST_SUBSCRIPTION, receiver) - .setExecutor(fakeExecutor) - .setCredentials(testCredentials) - .setChannelBuilder(testChannelBuilder); - } - - @SuppressWarnings("unchecked") - private void assertEquivalent(Collection expectedElems, Collection target) { - List remaining = new ArrayList(target.size()); - remaining.addAll(target); - - for (T expectedElem : expectedElems) { - if (!remaining.contains(expectedElem)) { - throw new AssertionError( - String.format("Expected element %s is not contained in %s", expectedElem, target)); - } - remaining.remove(expectedElem); - } - } - - @SuppressWarnings("unchecked") - private void assertEquivalentWithTransformation( - Collection expectedElems, Collection target, Function transform) { - List remaining = new ArrayList(target.size()); - remaining.addAll(target); - - for (E expectedElem : expectedElems) { - if (!remaining.contains(transform.apply(expectedElem))) { - throw new AssertionError( - String.format("Expected element %s is not contained in %s", expectedElem, target)); - } - remaining.remove(expectedElem); - } - } -} diff --git a/client/src/test/resources/log4j.xml b/client/src/test/resources/log4j.xml deleted file mode 100644 index d48bbabd..00000000 --- a/client/src/test/resources/log4j.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/client/src/test/resources/logging.properties b/client/src/test/resources/logging.properties deleted file mode 100644 index 5552719e..00000000 --- a/client/src/test/resources/logging.properties +++ /dev/null @@ -1,5 +0,0 @@ -handlers=java.util.logging.ConsoleHandler -.level=INFO -java.util.logging.ConsoleHandler.level=INFO -java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=[%1$tc] [%4$s] [%2$s] %5$s%6$s%n diff --git a/jms-light/configs/checkstyle/google_checks.xml b/jms-light/configs/checkstyle/google_checks.xml new file mode 100644 index 00000000..a9844747 --- /dev/null +++ b/jms-light/configs/checkstyle/google_checks.xml @@ -0,0 +1,218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jms-light/configs/checkstyle/rules.xml b/jms-light/configs/checkstyle/rules.xml deleted file mode 100644 index 1c9d2bde..00000000 --- a/jms-light/configs/checkstyle/rules.xml +++ /dev/null @@ -1,193 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/jms-light/pom.xml b/jms-light/pom.xml index 7360a379..c6a6d85a 100644 --- a/jms-light/pom.xml +++ b/jms-light/pom.xml @@ -16,9 +16,11 @@ org.apache.maven.plugins maven-compiler-plugin + 3.6.1 ${java.version} ${java.version} + -Xlint:all @@ -27,6 +29,13 @@ org.apache.maven.plugins maven-checkstyle-plugin 2.17 + + + com.puppycrawl.tools + checkstyle + 7.6 + + checkstyle-validation @@ -35,7 +44,7 @@ checkstyle - configs/checkstyle/rules.xml + configs/checkstyle/google_checks.xml configs/checkstyle/suppression.xml checkstyle.suppressions.file ${project.build.sourceDirectory} @@ -43,6 +52,7 @@ true true true + true @@ -121,6 +131,15 @@ + + + + org.apache.maven.plugins + maven-jxr-plugin + 2.5 + + + diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/AbstractMessageProducer.java b/jms-light/src/main/java/com/google/pubsub/jms/light/AbstractMessageProducer.java index 79c7998f..71091683 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/AbstractMessageProducer.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/AbstractMessageProducer.java @@ -12,8 +12,7 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractMessageProducer implements MessageProducer -{ +public abstract class AbstractMessageProducer implements MessageProducer { private final Destination destination; private final Session session; @@ -30,109 +29,91 @@ public abstract class AbstractMessageProducer implements MessageProducer * @param session is a jms session. * @param destination is a jms destination. */ - public AbstractMessageProducer(final Session session, final Destination destination) - { + public AbstractMessageProducer(final Session session, final Destination destination) { this.destination = destination; this.session = session; } - protected Session getSession() - { + protected Session getSession() { return session; } @Override - public void setDisableMessageID(final boolean disableMessageId) throws JMSException - { + public void setDisableMessageID(final boolean disableMessageId) throws JMSException { this.disableMessageId = disableMessageId; } @Override - public boolean getDisableMessageID() throws JMSException - { + public boolean getDisableMessageID() throws JMSException { return disableMessageId; } @Override - public void setDisableMessageTimestamp(final boolean disableTimestamp) throws JMSException - { + public void setDisableMessageTimestamp(final boolean disableTimestamp) throws JMSException { this.disableTimestamp = disableTimestamp; } @Override - public boolean getDisableMessageTimestamp() throws JMSException - { + public boolean getDisableMessageTimestamp() throws JMSException { return disableTimestamp; } @Override - public void setDeliveryMode(final int deliveryMode) throws JMSException - { + public void setDeliveryMode(final int deliveryMode) throws JMSException { this.deliveryMode = deliveryMode; } @Override - public int getDeliveryMode() throws JMSException - { + public int getDeliveryMode() throws JMSException { return deliveryMode; } @Override - public void setPriority(final int defaultPriority) throws JMSException - { + public void setPriority(final int defaultPriority) throws JMSException { this.priority = defaultPriority; } @Override - public int getPriority() throws JMSException - { + public int getPriority() throws JMSException { return priority; } @Override - public void setTimeToLive(final long timeToLive) throws JMSException - { + public void setTimeToLive(final long timeToLive) throws JMSException { this.timeToLive = timeToLive; } @Override - public long getTimeToLive() throws JMSException - { + public long getTimeToLive() throws JMSException { return timeToLive; } @Override - public void setDeliveryDelay(final long deliveryDelay) throws JMSException - { + public void setDeliveryDelay(final long deliveryDelay) throws JMSException { this.deliveryDelay = deliveryDelay; } @Override - public long getDeliveryDelay() throws JMSException - { + public long getDeliveryDelay() throws JMSException { return deliveryDelay; } @Override - public Destination getDestination() throws JMSException - { + public Destination getDestination() throws JMSException { return destination; } @Override - public void close() throws JMSException - { + public void close() throws JMSException { closed = true; } - protected boolean isClosed() - { + protected boolean isClosed() { return closed; } @Override - public void send(final Message message) throws JMSException - { + public void send(final Message message) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, null); } @@ -140,15 +121,13 @@ public void send(final Message message) throws JMSException public void send(final Message message, final int deliveryMode, final int priority, - final long timeToLive) throws JMSException - { + final long timeToLive) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, null); } @Override public void send(final Destination destination, - final Message message) throws JMSException - { + final Message message) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, null); } @@ -157,15 +136,13 @@ public void send(final Destination destination, final Message message, final int deliveryMode, final int priority, - final long timeToLive) throws JMSException - { + final long timeToLive) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, null); } @Override public void send(final Message message, - final CompletionListener completionListener) throws JMSException - { + final CompletionListener completionListener) throws JMSException { send(getDestination(), message, deliveryMode, priority, timeToLive, completionListener); } @@ -175,16 +152,14 @@ public void send(final Message message, final int deliveryMode, final int priority, final long timeToLive, - final CompletionListener completionListener) throws JMSException - { + final CompletionListener completionListener) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, completionListener); } @Override public void send(final Destination destination, final Message message, - final CompletionListener completionListener) throws JMSException - { + final CompletionListener completionListener) throws JMSException { send(destination, message, deliveryMode, priority, timeToLive, completionListener); } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnection.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnection.java index b4e5bbad..11a733e2 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnection.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnection.java @@ -5,6 +5,7 @@ import com.google.api.gax.grpc.ProviderManager; import com.google.common.collect.Sets; +import java.util.Set; import javax.jms.Connection; import javax.jms.ConnectionConsumer; import javax.jms.ConnectionMetaData; @@ -14,15 +15,13 @@ import javax.jms.ServerSessionPool; import javax.jms.Session; import javax.jms.Topic; -import java.util.Set; /** * Default PubSub {@link Connection} implementation. * * @author Maksym Prokhorenko */ -public class PubSubConnection implements Connection -{ +public class PubSubConnection implements Connection { private String clientId; private ExceptionListener exceptionListener; private Set sessions = Sets.newConcurrentHashSet(); @@ -34,29 +33,27 @@ public class PubSubConnection implements Connection /** * Default Connection constructor. * @param providerManager is a channel and executor container. Used by Publisher/Subscriber. - * @param flowControlSettings is a flow control: such as max outstanding messages and maximum outstanding bytes. + * @param flowControlSettings is a flow control: such as max outstanding messages and maximum + * outstanding bytes. * @param retrySettings is a retry logic configuration. */ public PubSubConnection( final ProviderManager providerManager, final FlowControlSettings flowControlSettings, - final RetrySettings retrySettings) - { + final RetrySettings retrySettings) { this.providerManager = providerManager; this.flowControlSettings = flowControlSettings; this.retrySettings = retrySettings; } @Override - public Session createSession(final boolean transacted, final int acknowledgeMode) throws JMSException - { + public Session createSession( + final boolean transacted, + final int acknowledgeMode) throws JMSException { final Session result; - if (transacted) - { + if (transacted) { throw new UnsupportedOperationException("Transactions is not supported yet."); - } - else - { + } else { result = createSession(acknowledgeMode); } @@ -64,16 +61,12 @@ public Session createSession(final boolean transacted, final int acknowledgeMode } @Override - public Session createSession(final int acknowledgeMode) throws JMSException - { + public Session createSession(final int acknowledgeMode) throws JMSException { final Session result; - if (Session.SESSION_TRANSACTED == acknowledgeMode) - { + if (Session.SESSION_TRANSACTED == acknowledgeMode) { throw new UnsupportedOperationException("Transactions is not supported yet."); - } - else - { + } else { result = new PubSubSession(this, false, acknowledgeMode); sessions.add(result); } @@ -82,58 +75,48 @@ public Session createSession(final int acknowledgeMode) throws JMSException } @Override - public Session createSession() throws JMSException - { + public Session createSession() throws JMSException { return createSession(Session.AUTO_ACKNOWLEDGE); } @Override - public String getClientID() throws JMSException - { + public String getClientID() throws JMSException { return clientId; } @Override - public void setClientID(final String clientId) throws JMSException - { + public void setClientID(final String clientId) throws JMSException { this.clientId = clientId; } @Override - public ConnectionMetaData getMetaData() throws JMSException - { + public ConnectionMetaData getMetaData() throws JMSException { return null; } @Override - public ExceptionListener getExceptionListener() throws JMSException - { + public ExceptionListener getExceptionListener() throws JMSException { return exceptionListener; } @Override - public void setExceptionListener(final ExceptionListener exceptionListener) throws JMSException - { + public void setExceptionListener(final ExceptionListener exceptionListener) throws JMSException { this.exceptionListener = exceptionListener; } @Override - public void start() throws JMSException - { + public void start() throws JMSException { } @Override - public void stop() throws JMSException - { + public void stop() throws JMSException { } @Override - public void close() throws JMSException - { - for (final Session session : sessions) - { + public void close() throws JMSException { + for (final Session session : sessions) { session.close(); } @@ -141,60 +124,53 @@ public void close() throws JMSException } @Override - public ConnectionConsumer createConnectionConsumer(final Destination destination, - final String messageSelector, - final ServerSessionPool serverSessionPool, - final int maxMessages) - throws JMSException - { + public ConnectionConsumer createConnectionConsumer( + final Destination destination, + final String messageSelector, + final ServerSessionPool serverSessionPool, + final int maxMessages) throws JMSException { return null; } @Override - public ConnectionConsumer createSharedConnectionConsumer(final Topic topic, - final String subscriptionName, - final String messageSelector, - final ServerSessionPool serverSessionPool, - final int maxMessages) - throws JMSException - { + public ConnectionConsumer createSharedConnectionConsumer( + final Topic topic, + final String subscriptionName, + final String messageSelector, + final ServerSessionPool serverSessionPool, + final int maxMessages) throws JMSException { return null; } @Override - public ConnectionConsumer createDurableConnectionConsumer(final Topic topic, - final String subscriptionName, - final String messageSelector, - final ServerSessionPool serverSessionPool, - final int maxMessages) - throws JMSException - { + public ConnectionConsumer createDurableConnectionConsumer( + final Topic topic, + final String subscriptionName, + final String messageSelector, + final ServerSessionPool serverSessionPool, + final int maxMessages) throws JMSException { return null; } @Override - public ConnectionConsumer createSharedDurableConnectionConsumer(final Topic topic, - final String subscriptionName, - final String messageSelector, - final ServerSessionPool serverSessionPool, - final int maxMessages) - throws JMSException - { + public ConnectionConsumer createSharedDurableConnectionConsumer( + final Topic topic, + final String subscriptionName, + final String messageSelector, + final ServerSessionPool serverSessionPool, + final int maxMessages) throws JMSException { return null; } - public FlowControlSettings getFlowControlSettings() - { + public FlowControlSettings getFlowControlSettings() { return flowControlSettings; } - public RetrySettings getRetrySettings() - { + public RetrySettings getRetrySettings() { return retrySettings; } - public ProviderManager getProviderManager() - { + public ProviderManager getProviderManager() { return providerManager; } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnectionFactory.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnectionFactory.java index 1386b74a..3b8aa147 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnectionFactory.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubConnectionFactory.java @@ -4,80 +4,74 @@ import com.google.api.gax.grpc.FlowControlSettings; import com.google.api.gax.grpc.ProviderManager; import com.google.common.base.MoreObjects; -import org.joda.time.Duration; - import javax.jms.Connection; import javax.jms.ConnectionFactory; import javax.jms.JMSContext; import javax.jms.JMSException; +import org.joda.time.Duration; /** * Default PubSub {@link ConnectionFactory} implementation. * * @author Maksym Prokhorenko */ -public class PubSubConnectionFactory implements ConnectionFactory -{ +public class PubSubConnectionFactory implements ConnectionFactory { private ProviderManager providerManager; private FlowControlSettings flowControlSettings; private RetrySettings retrySettings; @Override - public Connection createConnection() throws JMSException - { + public Connection createConnection() throws JMSException { return new PubSubConnection(providerManager, MoreObjects.firstNonNull(flowControlSettings, getDefaultFlowControllerSettings()), MoreObjects.firstNonNull(retrySettings, getDefaultRetrySettings())); } @Override - public Connection createConnection(final String userName, final String password) throws JMSException - { + public Connection createConnection( + final String userName, + final String password) throws JMSException { return null; } @Override - public JMSContext createContext() - { + public JMSContext createContext() { return null; } @Override - public JMSContext createContext(final String userName, final String password) - { + public JMSContext createContext(final String userName, final String password) { return null; } @Override - public JMSContext createContext(final String userName, final String password, final int sessionMode) - { + public JMSContext createContext( + final String userName, + final String password, + final int sessionMode) { return null; } @Override - public JMSContext createContext(final int sessionMode) - { + + public JMSContext createContext(final int sessionMode) { return null; } - public void setProviderManager(final ProviderManager providerManager) - { + public void setProviderManager(final ProviderManager providerManager) { this.providerManager = providerManager; } - public void setFlowControlSettings(final FlowControlSettings flowControlSettings) - { + public void setFlowControlSettings(final FlowControlSettings flowControlSettings) { this.flowControlSettings = flowControlSettings; } - public void setRetrySettings(final RetrySettings retrySettings) - { + public void setRetrySettings(final RetrySettings retrySettings) { this.retrySettings = retrySettings; } @SuppressWarnings("checkstyle:magicnumber") - protected RetrySettings getDefaultRetrySettings() - { + protected RetrySettings getDefaultRetrySettings() { return RetrySettings.newBuilder() .setTotalTimeout(Duration.standardSeconds(15L)) .setInitialRetryDelay(Duration.millis(200)) @@ -89,8 +83,7 @@ protected RetrySettings getDefaultRetrySettings() .build(); } - protected FlowControlSettings getDefaultFlowControllerSettings() - { + protected FlowControlSettings getDefaultFlowControllerSettings() { return FlowControlSettings.getDefaultInstance(); } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubMessageProducer.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubMessageProducer.java index d8c62c64..cbc7dc37 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubMessageProducer.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubMessageProducer.java @@ -7,6 +7,9 @@ import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.jms.CompletionListener; import javax.jms.Destination; import javax.jms.IllegalStateException; @@ -15,17 +18,13 @@ import javax.jms.Message; import javax.jms.Session; import javax.jms.Topic; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Default PubSub {@link javax.jms.MessageProducer} implementation. * * @author Maksym Prokhorenko */ -public class PubSubMessageProducer extends AbstractMessageProducer -{ +public class PubSubMessageProducer extends AbstractMessageProducer { private static final Logger LOGGER = Logger.getLogger(PubSubMessageProducer.class.getName()); private Publisher publisher; @@ -34,31 +33,28 @@ public class PubSubMessageProducer extends AbstractMessageProducer * @param session is a jms session. * @param destination is a jms destination. * @throws JMSException in case {@link PubSubMessageProducer} doesn't support destination - * or destination object fails to return topic/queue name. + * or destination object fails to return topic/queue name. */ - public PubSubMessageProducer(final Session session, - final Destination destination) - throws JMSException - { + public PubSubMessageProducer( + final Session session, + final Destination destination) throws JMSException { super(session, destination); publisher = createPublisher(destination); } @Override - public void send(final Destination destination, - final Message message, - final int deliveryMode, - final int priority, - final long timeToLive, - final CompletionListener completionListener) throws JMSException - { - if (isClosed()) - { + public void send( + final Destination destination, + final Message message, + final int deliveryMode, + final int priority, + final long timeToLive, + final CompletionListener completionListener) throws JMSException { + if (isClosed()) { throw new IllegalStateException("Producer has been closed."); } - if (!getDestination().equals(destination)) - { + if (!getDestination().equals(destination)) { throw new IllegalArgumentException("Destination [" + destination + "] is invalid. Expected [" + getDestination() + "]."); } @@ -69,49 +65,38 @@ public void send(final Destination destination, .build()); messageIdFuture.addCallback( - new RpcFutureCallback() - { - @Override public void onSuccess(final String messageId) - { + new RpcFutureCallback() { + @Override public void onSuccess(final String messageId) { LOGGER.fine(String.format("%s has been sent successfully.", messageId)); - if (null != completionListener) - { + if (null != completionListener) { completionListener.onCompletion(message); } } - @Override public void onFailure(final Throwable thrown) - { + @Override public void onFailure(final Throwable thrown) { LOGGER.log(Level.SEVERE, "Message sending error:", thrown); - if (null != completionListener) - { + if (null != completionListener) { completionListener.onException(message, (Exception) thrown); } } }); } - protected Publisher createPublisher(final Destination destination) throws JMSException - { + protected Publisher createPublisher(final Destination destination) throws JMSException { final Publisher result; - if (destination instanceof Topic) - { + if (destination instanceof Topic) { result = createPublisher((Topic) destination); - } - else - { + } else { throw new InvalidDestinationException("Unsupported destination."); } return result; } - protected Publisher createPublisher(final Topic topic) throws JMSException - { + protected Publisher createPublisher(final Topic topic) throws JMSException { final PubSubConnection connection = ((PubSubSession) getSession()).getConnection(); - try - { + try { return Publisher .newBuilder(TopicName.parse(topic.getTopicName())) .setChannelProvider(connection.getProviderManager()) @@ -119,9 +104,7 @@ protected Publisher createPublisher(final Topic topic) throws JMSException .setFlowControlSettings(connection.getFlowControlSettings()) .setRetrySettings(connection.getRetrySettings()) .build(); - } - catch (final IOException e) - { + } catch (final IOException e) { LOGGER.log(Level.SEVERE, "Can't create publisher.", e); throw new JMSException(e.getMessage()); } @@ -129,16 +112,12 @@ protected Publisher createPublisher(final Topic topic) throws JMSException @SuppressWarnings("PMD.AvoidCatchingGenericException") @Override - public synchronized void close() throws JMSException - { + public synchronized void close() throws JMSException { super.close(); - try - { + try { publisher.shutdown(); - } - catch (final Exception e) - { + } catch (final Exception e) { LOGGER.log(Level.SEVERE, "Can't close message producer.", e); throw new JMSException(e.getMessage()); } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubSession.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubSession.java index 8fcb94a4..6a69a9f6 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubSession.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubSession.java @@ -9,49 +9,45 @@ * * @author Maksym Prokhorenko */ -public class PubSubSession extends AbstractSessionBrowserCreator -{ +public class PubSubSession extends AbstractSessionBrowserCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public PubSubSession(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public PubSubSession( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public void commit() throws JMSException - { + public void commit() throws JMSException { } @Override - public void rollback() throws JMSException - { + public void rollback() throws JMSException { } @Override - public void close() throws JMSException - { + public void close() throws JMSException { closeProducers(); } @Override - public void recover() throws JMSException - { + public void recover() throws JMSException { } @Override - public void run() - { + public void run() { } @Override - public void unsubscribe(final String name) throws JMSException - { + public void unsubscribe(final String name) throws JMSException { } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnection.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnection.java new file mode 100644 index 00000000..85ec00dd --- /dev/null +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnection.java @@ -0,0 +1,60 @@ +package com.google.pubsub.jms.light; + +import com.google.api.gax.core.RetrySettings; +import com.google.api.gax.grpc.FlowControlSettings; +import com.google.api.gax.grpc.ProviderManager; + +import javax.jms.ConnectionConsumer; +import javax.jms.JMSException; +import javax.jms.ServerSessionPool; +import javax.jms.Topic; +import javax.jms.TopicConnection; +import javax.jms.TopicSession; + +/** + * Default PubSub {@link TopicConnectionFactory} implementation. + * + * @author Daiqian Zhang + */ +public class PubSubTopicConnection extends PubSubConnection implements TopicConnection { + + /** + * Default Connection constructor. + * @param providerManager is a channel and executor container. Used by Publisher/Subscriber. + * @param flowControlSettings is a flow control: such as max outstanding messages and maximum + * outstanding bytes. + * @param retrySettings is a retry logic configuration. + */ + public PubSubTopicConnection( + final ProviderManager providerManager, + final FlowControlSettings flowControlSettings, + final RetrySettings retrySettings) { + super(providerManager, flowControlSettings, retrySettings); + } + + @Override + public ConnectionConsumer createConnectionConsumer( + final Topic topic, + final String messageSelector, + final ServerSessionPool sessionPool, + final int maxMessages) throws JMSException { + return null; + } + + @Override + public ConnectionConsumer createDurableConnectionConsumer( + final Topic topic, + final String subscriptionName, + final String messageSelector, + final ServerSessionPool sessionPool, + final int maxMessages) throws JMSException { + return null; + } + + @Override + public TopicSession createTopicSession( + final boolean transacted, + final int acknowledgeMode) throws JMSException { + return null; + } +} diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java new file mode 100644 index 00000000..25a74fbf --- /dev/null +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicConnectionFactory.java @@ -0,0 +1,28 @@ +package com.google.pubsub.jms.light; + +import javax.jms.JMSException; +import javax.jms.TopicConnection; +import javax.jms.TopicConnectionFactory; + +/** + * Default PubSub {@link TopicConnectionFactory} implementation. + * + * @author Daiqian Zhang + */ +public class PubSubTopicConnectionFactory + extends PubSubConnectionFactory + implements TopicConnectionFactory { + + @Override + public TopicConnection createTopicConnection() throws JMSException { + return null; + } + + @Override + public TopicConnection createTopicConnection( + final String userName, + final String password) + throws JMSException { + return null; + } +} diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicPublisher.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicPublisher.java index 82a4c1a6..dea19c97 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicPublisher.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicPublisher.java @@ -10,55 +10,50 @@ * * @author Maksym Prokhorenko */ -public class PubSubTopicPublisher extends PubSubMessageProducer implements TopicPublisher -{ +public class PubSubTopicPublisher extends PubSubMessageProducer implements TopicPublisher { /** * Default constructor. * @param session is a jms session {@link javax.jms.Session}. * @param topic is a jms topic. * @throws JMSException in case {@link PubSubMessageProducer} doesn't support destination - * or destination object fails to return topic/queue name. + * or destination object fails to return topic/queue name. */ - public PubSubTopicPublisher(final PubSubSession session, final Topic topic) throws JMSException - { + public PubSubTopicPublisher(final PubSubSession session, final Topic topic) throws JMSException { super(session, topic); } @Override - public Topic getTopic() throws JMSException - { + public Topic getTopic() throws JMSException { return (Topic) getDestination(); } @Override - public void publish(final Message message) throws JMSException - { + public void publish(final Message message) throws JMSException { publish(getTopic(), message, getDeliveryMode(), getPriority(), getTimeToLive()); } @Override - public void publish(final Topic topic, final Message message) throws JMSException - { + public void publish(final Topic topic, final Message message) throws JMSException { publish(topic, message, getDeliveryMode(), getPriority(), getTimeToLive()); } @Override - public void publish(final Message message, - final int deliveryMode, - final int priority, - final long timeToLive) throws JMSException - { + public void publish( + final Message message, + final int deliveryMode, + final int priority, + final long timeToLive) throws JMSException { publish(getTopic(), message, deliveryMode, priority, timeToLive); } @Override - public void publish(final Topic topic, - final Message message, - final int deliveryMode, - final int priority, - final long timeToLive) throws JMSException - { + public void publish( + final Topic topic, + final Message message, + final int deliveryMode, + final int priority, + final long timeToLive) throws JMSException { send(topic, message, deliveryMode, priority, timeToLive); } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicSession.java b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicSession.java new file mode 100644 index 00000000..d4960393 --- /dev/null +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/PubSubTopicSession.java @@ -0,0 +1,61 @@ +package com.google.pubsub.jms.light; + +import com.google.pubsub.jms.light.destination.PubSubTemporaryTopic; + +import javax.jms.JMSException; +import javax.jms.Queue; +import javax.jms.QueueBrowser; +import javax.jms.TemporaryQueue; +import javax.jms.TemporaryTopic; + +/** + * Default PubSub {@link javax.jms.TopicSession} implementation. + * + * @author Daiqian Zhang + */ +class PubSubTopicSession extends PubSubSession { + + /** + * Default constructor. + * @param connection is a jms connection. + * @param transacted is an indicator whether the session in transacted mode. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. + */ + public PubSubTopicSession( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { + super(connection, transacted, acknowledgeMode); + } + + @Override + public TemporaryTopic createTemporaryTopic() { + return new PubSubTemporaryTopic(generateTemporaryTopicName()); + } + + @Override + public void unsubscribe(final String name) { + } + + @Override + public QueueBrowser createBrowser(final Queue queue) throws JMSException { + throw new JMSException("createBrowser can not be used in Pub/Sub messaging domain."); + } + + @Override + public Queue createQueue(final String queueName) throws JMSException { + throw new JMSException("createQueue can not be used in Pub/Sub messaging domain."); + } + + @Override + public TemporaryQueue createTemporaryQueue() throws JMSException { + throw new JMSException("createTemporaryQueue can not be used in Pub/Sub messaging domain."); + } + + private String generateTemporaryTopicName() { + // TODO make sure the generated string is unique to the connection. + return null; + } +} diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/consumer/PubSubMessageConsumer b/jms-light/src/main/java/com/google/pubsub/jms/light/consumer/PubSubMessageConsumer new file mode 100644 index 00000000..01a26518 --- /dev/null +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/consumer/PubSubMessageConsumer @@ -0,0 +1,147 @@ +package com.google.pubsub.jms.light.consumer; + +import com.google.cloud.pubsub.spi.v1.Subscriber; +import com.google.cloud.pubsub.spi.v1.SubscriptionAdminClient; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; + +import java.io.IOException; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.jms.IllegalStateException; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageConsumer; +import javax.jms.MessageListener; + +/** + * Default PubSub {@link MessageConsumer} abstraction. + * + * @author Raaja Prabhu U J + */ +public class PubSubMessageConsumer implements MessageConsumer { + private static final Logger LOGGER = Logger.getLogger(PubSubMessageConsumer.class.getName()); + + private final boolean isPush; + private final String project; + private final String topic; + + private boolean closed; + private MessageListener messageListener; + private Subscriber subscriber; + private SubscriptionAdminClient subscriptionAdminClient; + private Subscription subscription; + + /** + * Default PubSub's message consumer constructor. + * @param project name of a project to associate with JMS messages. + * @param topic is the topic within a project to subscribe to. + * @param isPush describes the message delivery method for the connection. + */ + public PubSubMessageConsumer(String project, String topic, boolean isPush) throws JMSException { + this.topic = topic; + this.isPush = isPush; + this.project = project; + this.closed = true; + if (!isPush) { + createSubscriptionAdminClient(); + } + } + + /** + * jms.MessageConsumer implementation. + */ + @Override + public void close() { + subscription = null; + closed = true; + } + + /** + * Google pub/sub doesn't support timeouts, so we aren't handling it for now. + */ + @Override + public Message receive(long timeoutInMillis) throws JMSException { + assertConsumerIsActive(); + assertNotPush(); + SubscriptionName subscriptionName = SubscriptionName.create(project, topic); + PullResponse response = subscriptionAdminClient.pull(subscriptionName, true, 1); + // TODO: convert PubSubMessage to JmsMessage and return it. + for (ReceivedMessage message : response.getReceivedMessagesList()) { + subscriptionAdminClient.acknowledge(subscriptionName, Collections.singletonList(message.getAckId())); + } + return null; + } + + @Override + public Message receive() throws JMSException { + return receive(-1); + } + + @Override + public Message receiveNoWait() throws JMSException { + return receive(0); + } + + @Override + public MessageListener getMessageListener() { + return messageListener; + } + + /** + * TODO: Create a webhook for Google's pub/sub to push messages. + */ + @Override + public void setMessageListener(MessageListener messageListener) throws JMSException { + assertConsumerIsActive(); + assertPush(); + if (messageListener.equals(this.messageListener)) { + return; + } + this.messageListener = messageListener; + this.closed = false; + } + + @Override + public String getMessageSelector() { + return null; + } + + + private void assertConsumerIsActive() throws IllegalStateException { + if (closed) { + throw new IllegalStateException("This consumer has been closed!"); + } + } + + private void assertPush() throws IllegalStateException { + if (!isPush) { + throw new IllegalStateException("This consumer is registered for pull-delivery of messages."); + } + } + + private void assertNotPush() throws IllegalStateException { + if (isPush) { + throw new IllegalStateException("This consumer is registered for push-delivery of messages."); + } + } + + private void createSubscriptionAdminClient() throws JMSException { + try { + TopicName topicName = TopicName.create(project, topic); + SubscriptionName subscriptionName = SubscriptionName.create(project, topic); + this.subscriptionAdminClient = SubscriptionAdminClient.create(); + this.subscription = subscriptionAdminClient + .createSubscription(subscriptionName, topicName, PushConfig.getDefaultInstance(), 0); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "Can't create SubscriptionAdminClient.", e); + throw new JMSException(e.getMessage()); + } + this.closed = false; + } +} diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubDestination.java b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubDestination.java index 6f4da5e1..8e6710fb 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubDestination.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubDestination.java @@ -1,14 +1,13 @@ package com.google.pubsub.jms.light.destination; -import javax.jms.Destination; import java.io.Serializable; +import javax.jms.Destination; /** * Default PubSub {@link Destination} implementation. * * @author Maksym Prokhorenko */ -public class PubSubDestination implements Destination, Serializable -{ +public class PubSubDestination implements Destination, Serializable { private static final long serialVersionUID = 1L; } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTemporaryTopic.java b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTemporaryTopic.java new file mode 100644 index 00000000..9ff46011 --- /dev/null +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTemporaryTopic.java @@ -0,0 +1,33 @@ +package com.google.pubsub.jms.light.destination; + +import javax.jms.JMSException; +import javax.jms.TemporaryTopic; + +/** + * Default PubSub {@link TemporaryTopic} implementation. + * + * @author Xiao (Frank) Yang + */ +public class PubSubTemporaryTopic extends PubSubDestination implements TemporaryTopic { + private final String topicName; + + /** + * Default PubSub Topic constructor. + * @param topicName is a pubsub topic name. + */ + public PubSubTemporaryTopic(final String topicName) { + super(); + this.topicName = topicName; + } + + @Override + public String getTopicName() throws JMSException { + return topicName; + } + + @Override + public void delete() throws JMSException { + // TODO: Throw JmsException if there are still subscribers on this topic. + // Waiting for the implementation of PubSubTopicSubscriber class. + } +} diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTopicDestination.java b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTopicDestination.java index 1e07d479..b2771802 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTopicDestination.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/destination/PubSubTopicDestination.java @@ -8,23 +8,20 @@ * * @author Maksym Prokhorenko */ -public class PubSubTopicDestination extends PubSubDestination implements Topic -{ +public class PubSubTopicDestination extends PubSubDestination implements Topic { private final String topicName; /** * Default PubSub Topic constructor. * @param topicName is a pubsub topic name. */ - public PubSubTopicDestination(final String topicName) - { + public PubSubTopicDestination(final String topicName) { super(); this.topicName = topicName; } @Override - public String getTopicName() throws JMSException - { + public String getTopicName() throws JMSException { return topicName; } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPropertyMessage.java b/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPropertyMessage.java index 3c6d9826..3e058c47 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPropertyMessage.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPropertyMessage.java @@ -2,148 +2,126 @@ import com.google.common.collect.Maps; -import javax.jms.JMSException; -import javax.jms.Message; import java.io.Serializable; import java.util.Enumeration; import java.util.Map; +import javax.jms.JMSException; +import javax.jms.Message; /** * Default implementation of property handling methods from {@link javax.jms.Message}. * * @author Maksym Prokhorenko */ -public abstract class AbstractPropertyMessage implements Message, Serializable -{ +public abstract class AbstractPropertyMessage implements Message, Serializable { private final Map properties = Maps.newHashMap(); @Override - public void clearProperties() throws JMSException - { + public void clearProperties() throws JMSException { properties.clear(); } @Override - public boolean propertyExists(final String name) throws JMSException - { + public boolean propertyExists(final String name) throws JMSException { return properties.containsKey(name); } @Override - public boolean getBooleanProperty(final String name) throws JMSException - { + public boolean getBooleanProperty(final String name) throws JMSException { return false; } @Override - public byte getByteProperty(final String name) throws JMSException - { + public byte getByteProperty(final String name) throws JMSException { return 0; } // short forced by protocol @SuppressWarnings("PMD.AvoidUsingShortType") @Override - public short getShortProperty(final String name) throws JMSException - { + public short getShortProperty(final String name) throws JMSException { return 0; } @Override - public int getIntProperty(final String name) throws JMSException - { + public int getIntProperty(final String name) throws JMSException { return 0; } @Override - public long getLongProperty(final String name) throws JMSException - { + public long getLongProperty(final String name) throws JMSException { return 0; } @Override - public float getFloatProperty(final String name) throws JMSException - { + public float getFloatProperty(final String name) throws JMSException { return 0; } @Override - public double getDoubleProperty(final String name) throws JMSException - { + public double getDoubleProperty(final String name) throws JMSException { return 0; } @Override - public String getStringProperty(final String name) throws JMSException - { + public String getStringProperty(final String name) throws JMSException { return null; } @Override - public Object getObjectProperty(final String name) throws JMSException - { + public Object getObjectProperty(final String name) throws JMSException { return null; } @Override - public Enumeration getPropertyNames() throws JMSException - { + public Enumeration getPropertyNames() throws JMSException { return null; } @Override - public void setBooleanProperty(final String name, final boolean value) throws JMSException - { + public void setBooleanProperty(final String name, final boolean value) throws JMSException { } @Override - public void setByteProperty(final String name, final byte value) throws JMSException - { + public void setByteProperty(final String name, final byte value) throws JMSException { } // short forced by protocol @SuppressWarnings("PMD.AvoidUsingShortType") @Override - public void setShortProperty(final String name, final short value) throws JMSException - { + public void setShortProperty(final String name, final short value) throws JMSException { } @Override - public void setIntProperty(final String name, final int value) throws JMSException - { + public void setIntProperty(final String name, final int value) throws JMSException { } @Override - public void setLongProperty(final String name, final long value) throws JMSException - { + public void setLongProperty(final String name, final long value) throws JMSException { } @Override - public void setFloatProperty(final String name, final float value) throws JMSException - { + public void setFloatProperty(final String name, final float value) throws JMSException { } @Override - public void setDoubleProperty(final String name, final double value) throws JMSException - { + public void setDoubleProperty(final String name, final double value) throws JMSException { } @Override - public void setStringProperty(final String name, final String value) throws JMSException - { + public void setStringProperty(final String name, final String value) throws JMSException { } @Override - public void setObjectProperty(final String name, final Object value) throws JMSException - { + public void setObjectProperty(final String name, final Object value) throws JMSException { } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPubSubMessage.java b/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPubSubMessage.java index 7ebb5a5d..1ddd839e 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPubSubMessage.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/message/AbstractPubSubMessage.java @@ -2,17 +2,16 @@ import com.google.common.base.Charsets; +import java.util.logging.Logger; import javax.jms.Destination; import javax.jms.JMSException; -import java.util.logging.Logger; /** * Default PubSub {@link javax.jms.Message} implementation. * * @author Maksym Prokhorenko */ -public abstract class AbstractPubSubMessage extends AbstractPropertyMessage -{ +public abstract class AbstractPubSubMessage extends AbstractPropertyMessage { private static final Logger LOGGER = Logger.getLogger(AbstractPubSubMessage.class.getName()); private boolean acknowledged; @@ -29,160 +28,132 @@ public abstract class AbstractPubSubMessage extends AbstractPropertyMessage private int jmsPriority; @Override - public void acknowledge() throws JMSException - { - if (acknowledged) - { + public void acknowledge() throws JMSException { + if (acknowledged) { //TODO: seems as exceptional state. LOGGER.warning("Message already acknowledged. [" + getJMSMessageID() + "]"); - } - else - { + } else { acknowledged = true; } } @Override - public String getJMSMessageID() throws JMSException - { + public String getJMSMessageID() throws JMSException { return jmsMessageId; } @Override - public void setJMSMessageID(final String id) throws JMSException - { + public void setJMSMessageID(final String id) throws JMSException { this.jmsMessageId = id; } @Override - public long getJMSTimestamp() throws JMSException - { + public long getJMSTimestamp() throws JMSException { return jmsTimestamp; } @Override - public void setJMSTimestamp(final long timestamp) throws JMSException - { + public void setJMSTimestamp(final long timestamp) throws JMSException { this.jmsTimestamp = timestamp; } @Override - public byte[] getJMSCorrelationIDAsBytes() throws JMSException - { + public byte[] getJMSCorrelationIDAsBytes() throws JMSException { return null == correlationId ? null : correlationId.getBytes(Charsets.UTF_8); } @Override - public void setJMSCorrelationIDAsBytes(final byte[] correlationId) throws JMSException - { + public void setJMSCorrelationIDAsBytes(final byte[] correlationId) throws JMSException { this.correlationId = null == correlationId ? null : new String(correlationId, Charsets.UTF_8); } @Override - public void setJMSCorrelationID(final String correlationId) throws JMSException - { + public void setJMSCorrelationID(final String correlationId) throws JMSException { this.correlationId = correlationId; } @Override - public String getJMSCorrelationID() throws JMSException - { + public String getJMSCorrelationID() throws JMSException { return correlationId; } @Override - public Destination getJMSReplyTo() throws JMSException - { + public Destination getJMSReplyTo() throws JMSException { return replyTo; } @Override - public void setJMSReplyTo(final Destination replyTo) throws JMSException - { + public void setJMSReplyTo(final Destination replyTo) throws JMSException { this.replyTo = replyTo; } @Override - public Destination getJMSDestination() throws JMSException - { + public Destination getJMSDestination() throws JMSException { return destination; } @Override - public void setJMSDestination(final Destination destination) throws JMSException - { + public void setJMSDestination(final Destination destination) throws JMSException { this.destination = destination; } @Override - public int getJMSDeliveryMode() throws JMSException - { + public int getJMSDeliveryMode() throws JMSException { return jmsDeliveryMode; } @Override - public void setJMSDeliveryMode(final int deliveryMode) throws JMSException - { + public void setJMSDeliveryMode(final int deliveryMode) throws JMSException { this.jmsDeliveryMode = deliveryMode; } @Override - public boolean getJMSRedelivered() throws JMSException - { + public boolean getJMSRedelivered() throws JMSException { return jmsRedelivered; } @Override - public void setJMSRedelivered(final boolean redelivered) throws JMSException - { + public void setJMSRedelivered(final boolean redelivered) throws JMSException { this.jmsRedelivered = redelivered; } @Override - public String getJMSType() throws JMSException - { + public String getJMSType() throws JMSException { return jmsType; } @Override - public void setJMSType(final String type) throws JMSException - { + public void setJMSType(final String type) throws JMSException { this.jmsType = type; } @Override - public long getJMSExpiration() throws JMSException - { + public long getJMSExpiration() throws JMSException { return jmsExpiration; } @Override - public void setJMSExpiration(final long expiration) throws JMSException - { + public void setJMSExpiration(final long expiration) throws JMSException { this.jmsExpiration = expiration; } @Override - public long getJMSDeliveryTime() throws JMSException - { + public long getJMSDeliveryTime() throws JMSException { return jmsDeliveryTime; } @Override - public void setJMSDeliveryTime(final long deliveryTime) throws JMSException - { + public void setJMSDeliveryTime(final long deliveryTime) throws JMSException { this.jmsDeliveryTime = deliveryTime; } @Override - public int getJMSPriority() throws JMSException - { + public int getJMSPriority() throws JMSException { return jmsPriority; } @Override - public void setJMSPriority(final int priority) throws JMSException - { + public void setJMSPriority(final int priority) throws JMSException { this.jmsPriority = priority; } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/message/PubSubTextMessage.java b/jms-light/src/main/java/com/google/pubsub/jms/light/message/PubSubTextMessage.java index 0ef875ce..77beab1e 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/message/PubSubTextMessage.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/message/PubSubTextMessage.java @@ -9,47 +9,38 @@ * * @author Maksym Prokhorenko */ -public class PubSubTextMessage extends AbstractPubSubMessage implements TextMessage -{ +public class PubSubTextMessage extends AbstractPubSubMessage implements TextMessage { private String payload; @Override - public void setText(final String payload) throws JMSException - { + public void setText(final String payload) throws JMSException { this.payload = payload; } @Override - public String getText() throws JMSException - { + public String getText() throws JMSException { return payload; } @Override - public void clearBody() throws JMSException - { + public void clearBody() throws JMSException { payload = null; } @Override - public T getBody(final Class clazz) throws JMSException - { - final String result; - if (isBodyAssignableTo(clazz)) - { - result = getText(); - } - else - { + public T getBody(final Class clazz) throws JMSException { + final T result; + if (isBodyAssignableTo(clazz)) { + result = clazz.cast(getText()); + } else { throw new MessageFormatException("Can't be assigned to " + clazz); } - return (T) result; + return result; } @Override - public boolean isBodyAssignableTo(final Class clazz) throws JMSException - { + public boolean isBodyAssignableTo(final Class clazz) throws JMSException { return clazz.equals(String.class); } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSession.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSession.java index 4a5a0bf8..da819479 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSession.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSession.java @@ -11,8 +11,7 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSession implements Session -{ +public abstract class AbstractSession implements Session { private final PubSubConnection connection; private final boolean transacted; private final int acknowledgeMode; @@ -22,43 +21,40 @@ public abstract class AbstractSession implements Session * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSession(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSession( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { this.connection = connection; this.transacted = transacted; this.acknowledgeMode = acknowledgeMode; } - public PubSubConnection getConnection() - { + public PubSubConnection getConnection() { return connection; } @Override - public boolean getTransacted() throws JMSException - { + public boolean getTransacted() throws JMSException { return transacted; } @Override - public int getAcknowledgeMode() throws JMSException - { + public int getAcknowledgeMode() throws JMSException { return acknowledgeMode; } @Override - public MessageListener getMessageListener() throws JMSException - { + public MessageListener getMessageListener() throws JMSException { return messageListener; } @Override - public void setMessageListener(final MessageListener listener) throws JMSException - { + public void setMessageListener(final MessageListener listener) throws JMSException { this.messageListener = listener; } - } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionBrowserCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionBrowserCreator.java index fce17d2e..de68acaf 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionBrowserCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionBrowserCreator.java @@ -11,29 +11,31 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionBrowserCreator extends AbstractSessionConsumerCreator -{ +public abstract class AbstractSessionBrowserCreator extends AbstractSessionConsumerCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionBrowserCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionBrowserCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public QueueBrowser createBrowser(final Queue queue) throws JMSException - { + public QueueBrowser createBrowser(final Queue queue) throws JMSException { return null; } @Override - public QueueBrowser createBrowser(final Queue queue, final String messageSelector) throws JMSException - { + public QueueBrowser createBrowser( + final Queue queue, + final String messageSelector) throws JMSException { return null; } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionConsumerCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionConsumerCreator.java index 6366d854..5e4c52c9 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionConsumerCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionConsumerCreator.java @@ -12,77 +12,85 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionConsumerCreator extends AbstractSessionSubscriberCreator -{ +public abstract class AbstractSessionConsumerCreator extends AbstractSessionSubscriberCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionConsumerCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionConsumerCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public MessageConsumer createConsumer(final Destination destination) throws JMSException - { + public MessageConsumer createConsumer(final Destination destination) throws JMSException { return null; } @Override - public MessageConsumer createConsumer(final Destination destination, final String messageSelector) throws JMSException - { + public MessageConsumer createConsumer( + final Destination destination, + final String messageSelector) throws JMSException { return null; } @Override - public MessageConsumer createConsumer(final Destination destination, final String messageSelector, final boolean noLocal) throws JMSException - { + public MessageConsumer createConsumer( + final Destination destination, + final String messageSelector, + final boolean noLocal) throws JMSException { return null; } @Override - public MessageConsumer createSharedConsumer(final Topic topic, final String sharedSubscriptionName) throws JMSException - { + public MessageConsumer createSharedConsumer( + final Topic topic, + final String sharedSubscriptionName) throws JMSException { return null; } @Override - public MessageConsumer createSharedConsumer(final Topic topic, - final String sharedSubscriptionName, - final String messageSelector) throws JMSException - { + public MessageConsumer createSharedConsumer( + final Topic topic, + final String sharedSubscriptionName, + final String messageSelector) throws JMSException { return null; } @Override - public MessageConsumer createDurableConsumer(final Topic topic, final String name) throws JMSException - { + public MessageConsumer createDurableConsumer( + final Topic topic, + final String name) throws JMSException { return null; } @Override - public MessageConsumer createDurableConsumer(final Topic topic, - final String name, - final String messageSelector, - final boolean noLocal) throws JMSException - { + public MessageConsumer createDurableConsumer( + final Topic topic, + final String name, + final String messageSelector, + final boolean noLocal) throws JMSException { return null; } @Override - public MessageConsumer createSharedDurableConsumer(final Topic topic, final String name) throws JMSException - { + public MessageConsumer createSharedDurableConsumer( + final Topic topic, + final String name) throws JMSException { return null; } @Override - public MessageConsumer createSharedDurableConsumer(final Topic topic, final String name, final String messageSelector) throws JMSException - { + public MessageConsumer createSharedDurableConsumer( + final Topic topic, + final String name, + final String messageSelector) throws JMSException { return null; } - } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionMessageCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionMessageCreator.java index 3e9c8bb3..1c311101 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionMessageCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionMessageCreator.java @@ -3,6 +3,7 @@ import com.google.pubsub.jms.light.PubSubConnection; import com.google.pubsub.jms.light.message.PubSubTextMessage; +import java.io.Serializable; import javax.jms.BytesMessage; import javax.jms.JMSException; import javax.jms.MapMessage; @@ -10,73 +11,66 @@ import javax.jms.ObjectMessage; import javax.jms.StreamMessage; import javax.jms.TextMessage; -import java.io.Serializable; /** * Default implementation of {@link javax.jms.Session} message creations. * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionMessageCreator extends AbstractSession -{ +public abstract class AbstractSessionMessageCreator extends AbstractSession { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionMessageCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionMessageCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public BytesMessage createBytesMessage() throws JMSException - { + public BytesMessage createBytesMessage() throws JMSException { return null; } @Override - public MapMessage createMapMessage() throws JMSException - { + public MapMessage createMapMessage() throws JMSException { return null; } @Override - public Message createMessage() throws JMSException - { + public Message createMessage() throws JMSException { return null; } @Override - public ObjectMessage createObjectMessage() throws JMSException - { + public ObjectMessage createObjectMessage() throws JMSException { return null; } @Override - public ObjectMessage createObjectMessage(final Serializable object) throws JMSException - { + public ObjectMessage createObjectMessage(final Serializable object) throws JMSException { return null; } @Override - public StreamMessage createStreamMessage() throws JMSException - { + public StreamMessage createStreamMessage() throws JMSException { return null; } @Override - public TextMessage createTextMessage() throws JMSException - { + public TextMessage createTextMessage() throws JMSException { return new PubSubTextMessage(); } @Override - public TextMessage createTextMessage(final String text) throws JMSException - { + public TextMessage createTextMessage(final String text) throws JMSException { final TextMessage message = createTextMessage(); message.setText(text); diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionProducerCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionProducerCreator.java index f2fee5e2..b7669142 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionProducerCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionProducerCreator.java @@ -4,44 +4,41 @@ import com.google.pubsub.jms.light.PubSubConnection; import com.google.pubsub.jms.light.PubSubMessageProducer; +import java.util.Set; import javax.jms.Destination; import javax.jms.JMSException; import javax.jms.MessageProducer; -import java.util.Set; /** * Default implementation of {@link javax.jms.Session} producer creations. * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionProducerCreator extends AbstractSessionQueueCreator -{ +public abstract class AbstractSessionProducerCreator extends AbstractSessionQueueCreator { private final Set producers = Sets.newConcurrentHashSet(); /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionProducerCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionProducerCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public MessageProducer createProducer(final Destination destination) - throws JMSException - { - + public MessageProducer createProducer(final Destination destination) throws JMSException { return new PubSubMessageProducer(this, destination); } - protected void closeProducers() throws JMSException - { - for (final MessageProducer producer : producers) - { + protected void closeProducers() throws JMSException { + for (final MessageProducer producer : producers) { producer.close(); } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionQueueCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionQueueCreator.java index 3c40ecdd..86b39036 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionQueueCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionQueueCreator.java @@ -11,29 +11,29 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionQueueCreator extends AbstractSessionTopicCreator -{ +public abstract class AbstractSessionQueueCreator extends AbstractSessionTopicCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionQueueCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionQueueCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public Queue createQueue(final String queueName) throws JMSException - { + public Queue createQueue(final String queueName) throws JMSException { return null; } @Override - public TemporaryQueue createTemporaryQueue() throws JMSException - { + public TemporaryQueue createTemporaryQueue() throws JMSException { return null; } } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionSubscriberCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionSubscriberCreator.java index 1d367339..a6436166 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionSubscriberCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionSubscriberCreator.java @@ -11,33 +11,35 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionSubscriberCreator extends AbstractSessionProducerCreator -{ +public abstract class AbstractSessionSubscriberCreator extends AbstractSessionProducerCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionSubscriberCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionSubscriberCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public TopicSubscriber createDurableSubscriber(final Topic topic, final String name) throws JMSException - { + public TopicSubscriber createDurableSubscriber( + final Topic topic, + final String name) throws JMSException { return null; } @Override - public TopicSubscriber createDurableSubscriber(final Topic topic, - final String name, - final String messageSelector, - final boolean noLocal) throws JMSException - { + public TopicSubscriber createDurableSubscriber( + final Topic topic, + final String name, + final String messageSelector, + final boolean noLocal) throws JMSException { return null; } - } diff --git a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionTopicCreator.java b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionTopicCreator.java index b6dea17c..b56e6e06 100644 --- a/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionTopicCreator.java +++ b/jms-light/src/main/java/com/google/pubsub/jms/light/session/AbstractSessionTopicCreator.java @@ -11,29 +11,29 @@ * * @author Maksym Prokhorenko */ -public abstract class AbstractSessionTopicCreator extends AbstractSessionMessageCreator -{ +public abstract class AbstractSessionTopicCreator extends AbstractSessionMessageCreator { /** * Default constructor. * @param connection is a jms connection. * @param transacted is an indicator whether the session in transacted mode. - * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, - * {@link javax.jms.Session#SESSION_TRANSACTED}. + * @param acknowledgeMode is an acknowledgement mode {@link javax.jms.Session#AUTO_ACKNOWLEDGE}, + * {@link javax.jms.Session#CLIENT_ACKNOWLEDGE}, + * {@link javax.jms.Session#SESSION_TRANSACTED}. */ - public AbstractSessionTopicCreator(final PubSubConnection connection, final boolean transacted, final int acknowledgeMode) - { + public AbstractSessionTopicCreator( + final PubSubConnection connection, + final boolean transacted, + final int acknowledgeMode) { super(connection, transacted, acknowledgeMode); } @Override - public Topic createTopic(final String topicName) throws JMSException - { + public Topic createTopic(final String topicName) throws JMSException { return null; } @Override - public TemporaryTopic createTemporaryTopic() throws JMSException - { + public TemporaryTopic createTemporaryTopic() throws JMSException { return null; } } diff --git a/jms-light/src/test/java/com/google/pubsub/jms/light/integration/BaseIntegrationTest.java b/jms-light/src/test/java/com/google/pubsub/jms/light/integration/BaseIntegrationTest.java index 4267cc31..a2cb5e12 100644 --- a/jms-light/src/test/java/com/google/pubsub/jms/light/integration/BaseIntegrationTest.java +++ b/jms-light/src/test/java/com/google/pubsub/jms/light/integration/BaseIntegrationTest.java @@ -6,16 +6,15 @@ import com.google.cloud.pubsub.deprecated.TopicInfo; import com.google.cloud.pubsub.deprecated.testing.LocalPubSubHelper; import com.google.common.base.Joiner; -import org.joda.time.Duration; -import org.junit.AfterClass; -import org.junit.BeforeClass; import java.io.IOException; import java.util.concurrent.TimeoutException; import java.util.logging.Logger; +import org.joda.time.Duration; +import org.junit.AfterClass; +import org.junit.BeforeClass; -public class BaseIntegrationTest -{ +public class BaseIntegrationTest { private static final Logger LOGGER = Logger.getLogger(BaseIntegrationTest.class.getName()); public static String PROJECT_ID = "jms-light"; @@ -27,9 +26,11 @@ public class BaseIntegrationTest private static LocalPubSubHelper EMBEDDED_PUBSUB_SERVICE; private static Subscription SUBSCRIPTION; + /** + * This method gets called before every test run and sets up a local pub/sub emulator. + */ @BeforeClass - public static void startPubSub() throws IOException, InterruptedException - { + public static void startPubSub() throws IOException, InterruptedException { EMBEDDED_PUBSUB_SERVICE = LocalPubSubHelper.create(); EMBEDDED_SERVICE_PORT = EMBEDDED_PUBSUB_SERVICE.getPort(); @@ -50,23 +51,19 @@ public static void startPubSub() throws IOException, InterruptedException } @AfterClass - public static void shutdownPubSub() throws InterruptedException, TimeoutException, IOException - { + public static void shutdownPubSub() throws InterruptedException, TimeoutException, IOException { EMBEDDED_PUBSUB_SERVICE.stop(Duration.standardSeconds(10L)); } - public static String getServiceHost() - { + public static String getServiceHost() { return EMBEDDED_SERVICE_HOST; } - public static int getServicePort() - { + public static int getServicePort() { return EMBEDDED_SERVICE_PORT; } - public static Subscription getServiceSubscription() - { + public static Subscription getServiceSubscription() { return SUBSCRIPTION; } } diff --git a/jms-light/src/test/java/com/google/pubsub/jms/light/integration/PublisherIntegrationTest.java b/jms-light/src/test/java/com/google/pubsub/jms/light/integration/PublisherIntegrationTest.java index f766ded1..69d62e71 100644 --- a/jms-light/src/test/java/com/google/pubsub/jms/light/integration/PublisherIntegrationTest.java +++ b/jms-light/src/test/java/com/google/pubsub/jms/light/integration/PublisherIntegrationTest.java @@ -1,5 +1,10 @@ package com.google.pubsub.jms.light.integration; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; + import com.google.api.gax.grpc.FixedChannelProvider; import com.google.api.gax.grpc.InstantiatingExecutorProvider; import com.google.api.gax.grpc.ProviderManager; @@ -9,11 +14,11 @@ import com.google.pubsub.jms.light.PubSubConnectionFactory; import com.google.pubsub.jms.light.destination.PubSubTopicDestination; import io.grpc.ManagedChannelBuilder; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.logging.Logger; import javax.jms.Connection; import javax.jms.ConnectionFactory; import javax.jms.JMSException; @@ -21,33 +26,27 @@ import javax.jms.Session; import javax.jms.TextMessage; import javax.jms.Topic; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.logging.Logger; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.awaitility.Awaitility.await; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasSize; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class PublisherIntegrationTest extends BaseIntegrationTest -{ +public class PublisherIntegrationTest extends BaseIntegrationTest { private static final Logger LOGGER = Logger.getLogger(PublisherIntegrationTest.class.getName()); private List toSend = Lists.newArrayList( - "\"Mystery creates wonder and wonder is the basis of man's desire to understand.\" -- Neil Armstrong", + "\"Mystery creates wonder and wonder is the basis of man's desire to understand.\" " + + "-- Neil Armstrong", "\"A-OK full go.\" -- Alan B. Shepard", "\"Houston, Tranquillity Base here. The Eagle has landed.\" -- Neil Armstrong" ); private ConnectionFactory connectionFactory = new PubSubConnectionFactory(); - private Topic topic = new PubSubTopicDestination(String.format("projects/%s/topics/%s", PROJECT_ID, TOPIC_NAME)); + private Topic topic = new PubSubTopicDestination( + String.format("projects/%s/topics/%s", PROJECT_ID, TOPIC_NAME)); @Before - public void setUp() throws IOException - { + public void setUp() throws IOException { final PubSubConnectionFactory factory = (PubSubConnectionFactory) connectionFactory; factory.setProviderManager(createProviderManager()); } @@ -56,14 +55,11 @@ public void setUp() throws IOException * Simple producer test. Creates JMS connection/session/producer/message and send it to PubSub. */ @Test - public void sunnyDayPublish() throws JMSException, IOException - { - try (final Connection connection = connectionFactory.createConnection()) - { + public void sunnyDayPublish() throws JMSException, IOException { + try (final Connection connection = connectionFactory.createConnection()) { final Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); final MessageProducer producer = session.createProducer(topic); - for (final String text : toSend) - { + for (final String text : toSend) { final TextMessage wowMessage = session.createTextMessage(text); producer.send(wowMessage); } @@ -76,38 +72,40 @@ public void sunnyDayPublish() throws JMSException, IOException // wait till all messages received. await().atMost(1, SECONDS).until( - new Callable>() - { - @Override public List call() throws Exception {return messageProcessor.getReceivedMessages();} + new Callable>() { + @Override public List call() throws Exception { + return messageProcessor.getReceivedMessages(); + } }, hasSize(greaterThanOrEqualTo(toSend.size()))); } - private ProviderManager createProviderManager() - { + private ProviderManager createProviderManager() { return ProviderManager.newBuilder() .setChannelProvider( FixedChannelProvider.create( - ManagedChannelBuilder.forAddress(getServiceHost(), getServicePort()).usePlaintext(true).build())) - .setExecutorProvider(InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()) + ManagedChannelBuilder.forAddress( + getServiceHost(), + getServicePort()).usePlaintext(true).build())) + .setExecutorProvider( + InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()) .build(); } - static class WowMessageProcessor implements PubSub.MessageProcessor - { + static class WowMessageProcessor implements PubSub.MessageProcessor { final List received = Lists.newArrayList(); @Override - public void process(final Message message) throws Exception - { + public void process(final Message message) throws Exception { received.add(message); LOGGER.info( - String.format("Received: [id: %s, payload: %s]", message.getId(), message.getPayloadAsString())); + String.format("Received: [id: %s, payload: %s]", + message.getId(), + message.getPayloadAsString())); } - List getReceivedMessages() - { + List getReceivedMessages() { return Collections.unmodifiableList(received); } } diff --git a/kafka-connector/README.md b/kafka-connector/README.md index 1fcf9740..b01e4352 100644 --- a/kafka-connector/README.md +++ b/kafka-connector/README.md @@ -1,9 +1,7 @@ ### Introduction -The CloudPubSubConnector is a connector to be used with [Kafka Connect] -(http://kafka.apache.org/documentation.html#connect) to publish messages from -[Kafka](http://kafka.apache.org) to [Google Cloud Pub/Sub] -(https://cloud.google.com/pubsub/) and vice versa. CloudPubSubConnector provides +The CloudPubSubConnector is a connector to be used with [Kafka Connect](http://kafka.apache.org/documentation.html#connect) to publish messages from +[Kafka](http://kafka.apache.org) to [Google Cloud Pub/Sub](https://cloud.google.com/pubsub/) and vice versa. CloudPubSubConnector provides both a sink connector (to copy messages from Kafka to Cloud Pub/Sub) and a source connector (to copy messages from Cloud Pub/Sub to Kafka). @@ -56,8 +54,14 @@ The resulting jar is at target/cps-kafka-connector.jar. 3. Create an appropriate configuration for your Kafka connect instance. More information on the configuration for Kafka connect can be found in the - [Kafka Users Guide] - (http://kafka.apache.org/documentation.html#connect_running). + [Kafka Users Guide](http://kafka.apache.org/documentation.html#connect_running). + +4. If running the Kafka Connector behind a proxy, you need to export the + KAFKA_OPTS variable with options for connecting around the proxy. You can + export this variable as part of a shell script in order ot make it easier. + Here is an example: + + `export KAFKA_OPTS="-Dhttp.proxyHost= -Dhttp.proxyPort= -Dhttps.proxyHost= -Dhttps.proxyPort="` ### CloudPubSubConnector Configs @@ -132,3 +136,4 @@ from a Pubsub message into a SourceRecord with a relevant Schema. * In these cases, to carry forward the structure of data stored in attributes, we recommend using a converter that can represent a struct schema type in a useful way, e.g. JsonConverter. + diff --git a/kafka-connector/pom.xml b/kafka-connector/pom.xml index f535850a..d34374ed 100644 --- a/kafka-connector/pom.xml +++ b/kafka-connector/pom.xml @@ -16,22 +16,23 @@ junit junit 4.12 + test io.grpc grpc-all - 0.15.0 + 1.6.1 io.netty netty-tcnative-boringssl-static - 1.1.33.Fork14 - linux-x86_64 + 2.0.3.Final + ${os.detected.classifier} com.google.apis google-api-services-pubsub - v1-rev7-1.21.0 + v1-rev360-1.23.0 com.google.guava @@ -42,27 +43,24 @@ com.google.api.grpc grpc-google-pubsub-v1 - 0.0.9 + 0.1.1 com.google.auth google-auth-library-oauth2-http - 0.3.0 - - - org.apache.kafka - kafka_2.11 - 0.10.1.0 + 0.8.0 org.apache.kafka connect-api - 0.10.1.0 + 0.10.2.0 + provided org.mockito mockito-core 1.10.19 + test @@ -108,6 +106,7 @@ com.fasterxml.jackson.core:jackson-core:jar:* + com.google.guava:* @@ -139,7 +138,7 @@ --> com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:0.15.0:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.6.1:exe:${os.detected.classifier} ${project.basedir}/src/main/proto diff --git a/kafka-connector/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java b/kafka-connector/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java index c3412a2a..47e30525 100644 --- a/kafka-connector/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java +++ b/kafka-connector/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java @@ -38,7 +38,6 @@ import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTask; -import org.mockito.internal.matchers.Null; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,6 +158,10 @@ public void put(Collection sinkRecords) { } private ByteString handleValue(Schema schema, Object value, Map attributes) { + if (schema == null) { + String str = value.toString(); + return ByteString.copyFromUtf8(str); + } Schema.Type t = schema.type(); switch (t) { case INT8: diff --git a/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java b/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java index 6c21baad..1c9ef97b 100644 --- a/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java +++ b/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java @@ -199,7 +199,7 @@ public void verifySubscription(String cpsProject, String cpsSubscription) { stub.getSubscription(request).get(); } catch (Exception e) { throw new ConnectException( - "The subscription " + cpsSubscription + " does not exist for the project" + cpsProject); + "Error verifying the subscription " + cpsSubscription + " for project " + cpsProject, e); } } diff --git a/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java b/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java index 5db1f6b0..ea03f1a6 100644 --- a/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java +++ b/kafka-connector/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java @@ -28,7 +28,6 @@ import com.google.pubsub.v1.PullRequest; import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -129,7 +128,7 @@ public List poll() throws InterruptedException { Map messageAttributes = message.getAttributes(); String key = messageAttributes.get(kafkaMessageKeyAttribute); ByteString messageData = message.getData(); - ByteBuffer messageBytes = messageData.asReadOnlyByteBuffer(); + byte[] messageBytes = messageData.toByteArray(); boolean hasAttributes = messageAttributes.size() > 1 || (messageAttributes.size() > 0 && key == null); @@ -165,7 +164,7 @@ record = null, kafkaTopic, selectPartition(key, value), - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, key, valueSchema, value); @@ -176,7 +175,7 @@ record = null, kafkaTopic, selectPartition(key, messageBytes), - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, key, Schema.BYTES_SCHEMA, messageBytes); @@ -185,36 +184,35 @@ record = } return sourceRecords; } catch (Exception e) { - // Kafka Connect suppresses any indication of an InterruptedException - // so we have to throw a RuntimeException. - throw new RuntimeException(e.getMessage()); + log.info("Error while retrieving records, treating as an empty poll. " + e); + return new ArrayList<>(); } } /** - * Attempt to ack all ids in {@link #ackIds}. If the ack request was unsuccessful then do not - * clear the list of acks. Instead, wait for the next call to this function to ack those ids. + * Attempt to ack all ids in {@link #ackIds}. Acks are best-effort, so if acking fails, messages + * may be delivered multiple times to Kafka. */ private void ackMessages() { if (ackIds.size() != 0) { - AcknowledgeRequest request = - AcknowledgeRequest.newBuilder() - .setSubscription(cpsSubscription) - .addAllAckIds(ackIds) - .build(); - ListenableFuture response = subscriber.ackMessages(request); + AcknowledgeRequest.Builder requestBuilder = AcknowledgeRequest.newBuilder() + .setSubscription(cpsSubscription); + synchronized (ackIds) { + requestBuilder.addAllAckIds(ackIds); + ackIds.clear(); + } + ListenableFuture response = subscriber.ackMessages(requestBuilder.build()); Futures.addCallback( response, new FutureCallback() { @Override public void onSuccess(Empty result) { log.trace("Successfully acked a set of messages."); - ackIds.clear(); } @Override public void onFailure(Throwable t) { - log.error("An exception occurred acking messages. Will try to ack messages again."); + log.error("An exception occurred acking messages: " + t); } }); } @@ -223,9 +221,9 @@ public void onFailure(Throwable t) { /** Return the partition a message should go to based on {@link #kafkaPartitionScheme}. */ private int selectPartition(Object key, Object value) { if (kafkaPartitionScheme.equals(PartitionScheme.HASH_KEY)) { - return key == null ? 0 : key.hashCode() % kafkaPartitions; + return key == null ? 0 : Math.abs(key.hashCode()) % kafkaPartitions; } else if (kafkaPartitionScheme.equals(PartitionScheme.HASH_VALUE)) { - return value.hashCode() % kafkaPartitions; + return Math.abs(value.hashCode()) % kafkaPartitions; } else { currentRoundRobinPartition = ++currentRoundRobinPartition % kafkaPartitions; return currentRoundRobinPartition; diff --git a/kafka-connector/src/test/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTaskTest.java b/kafka-connector/src/test/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTaskTest.java index ca189e4a..55402b25 100644 --- a/kafka-connector/src/test/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTaskTest.java +++ b/kafka-connector/src/test/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTaskTest.java @@ -166,6 +166,16 @@ record = new SinkRecord(null, -1, null, null, schema, null, -1); } catch (DataException e) { } // Expected, pass. } + @Test + public void testNullSchema() { + task.start(props); + String val = "I have no schema"; + SinkRecord record = new SinkRecord(null, -1, null, null, null, val, -1); + List list = new ArrayList<>(); + list.add(record); + task.put(list); + } + /** * Tests that if there are not enough messages buffered, publisher.publish() is not invoked. */ diff --git a/kafka-connector/src/test/java/com/google/pubsub/kafka/source/CloudPubSubSourceTaskTest.java b/kafka-connector/src/test/java/com/google/pubsub/kafka/source/CloudPubSubSourceTaskTest.java index 8c457fff..00bcc71b 100644 --- a/kafka-connector/src/test/java/com/google/pubsub/kafka/source/CloudPubSubSourceTaskTest.java +++ b/kafka-connector/src/test/java/com/google/pubsub/kafka/source/CloudPubSubSourceTaskTest.java @@ -15,6 +15,7 @@ //////////////////////////////////////////////////////////////////////////////// package com.google.pubsub.kafka.source; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; @@ -34,10 +35,10 @@ import com.google.pubsub.v1.PullRequest; import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; -import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -60,7 +61,7 @@ public class CloudPubSubSourceTaskTest { private static final String KAFKA_MESSAGE_KEY_ATTRIBUTE_VALUE = "jumped"; private static final String KAFKA_PARTITIONS = "3"; private static final ByteString CPS_MESSAGE = ByteString.copyFromUtf8("over"); - private static final ByteBuffer KAFKA_VALUE = CPS_MESSAGE.asReadOnlyByteBuffer(); + private static final byte[] KAFKA_VALUE = CPS_MESSAGE.toByteArray(); private static final String ACK_ID1 = "ackID1"; private static final String ACK_ID2 = "ackID2"; private static final String ACK_ID3 = "ackID3"; @@ -70,6 +71,31 @@ public class CloudPubSubSourceTaskTest { private Map props; private CloudPubSubSubscriber subscriber; + /** + * Compare two SourceRecords. This is necessary because the records' values contain a byte[] and + * the .equals on a SourceRecord does not take this into account. + */ + public void assertRecordsEqual(SourceRecord sr1, SourceRecord sr2) { + assertEquals(sr1.key(), sr2.key()); + assertEquals(sr1.keySchema(), sr2.keySchema()); + assertEquals(sr1.valueSchema(), sr2.valueSchema()); + assertEquals(sr1.topic(), sr2.topic()); + + if (sr1.valueSchema() == Schema.BYTES_SCHEMA) { + assertArrayEquals((byte[])sr1.value(), (byte[])sr2.value()); + } else { + for(Field f : sr1.valueSchema().fields()) { + if (f.name().equals(ConnectorUtils.KAFKA_MESSAGE_CPS_BODY_FIELD)) { + assertArrayEquals(((Struct)sr1.value()).getBytes(f.name()), + ((Struct)sr2.value()).getBytes(f.name())); + } else { + assertEquals(((Struct)sr1.value()).getString(f.name()), + ((Struct)sr2.value()).getString(f.name())); + } + } + } + } + @Before public void setup() { subscriber = mock(CloudPubSubSubscriber.class, RETURNS_DEEP_STUBS); @@ -121,7 +147,7 @@ public void testPollInRegularCase() throws Exception { /** - * Tests that when a call to ackMessages() fails, that the message is not sent again to Kafka if + * Tests that when a call to ackMessages() fails, that the message is redelivered to Kafka if * the message is received again by Cloud Pub/Sub. Also tests that ack ids are added properly if * the ack id has not been seen before. */ @@ -140,7 +166,7 @@ public void testPollWithDuplicateReceivedMessages() throws Exception { ListenableFuture failedFuture = Futures.immediateFailedFuture(new Throwable()); when(subscriber.ackMessages(any(AcknowledgeRequest.class))).thenReturn(failedFuture); result = task.poll(); - assertEquals(1, result.size()); + assertEquals(2, result.size()); verify(subscriber, times(1)).ackMessages(any(AcknowledgeRequest.class)); } @@ -164,11 +190,11 @@ public void testPollWithNoMessageKeyAttribute() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); - assertEquals(expected, result.get(0)); + assertRecordsEqual(expected, result.get(0)); } /** @@ -192,11 +218,11 @@ public void testPollWithMessageKeyAttribute() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, KAFKA_MESSAGE_KEY_ATTRIBUTE_VALUE, Schema.BYTES_SCHEMA, KAFKA_VALUE); - assertEquals(expected, result.get(0)); + assertRecordsEqual(expected, result.get(0)); } /** @@ -232,11 +258,11 @@ public void testPollWithMultipleAttributes() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, KAFKA_MESSAGE_KEY_ATTRIBUTE_VALUE, expectedSchema, expectedValue); - assertEquals(expected, result.get(0)); + assertRecordsEqual(expected, result.get(0)); } /** @@ -268,7 +294,7 @@ public void testPollWithPartitionSchemeHashKey() throws Exception { null, KAFKA_TOPIC, KAFKA_MESSAGE_KEY_ATTRIBUTE_VALUE.hashCode() % Integer.parseInt(KAFKA_PARTITIONS), - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, KAFKA_MESSAGE_KEY_ATTRIBUTE_VALUE, Schema.BYTES_SCHEMA, KAFKA_VALUE); @@ -278,13 +304,13 @@ public void testPollWithPartitionSchemeHashKey() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); - assertEquals(expectedForMessageWithKey, result.get(0)); - assertEquals(expectedForMessageWithoutKey.value(), result.get(1).value()); + assertRecordsEqual(expectedForMessageWithKey, result.get(0)); + assertArrayEquals((byte[])expectedForMessageWithoutKey.value(), (byte[])result.get(1).value()); } /** Tests that the correct partition is assigned when the partition scheme is "hash_value". */ @@ -306,11 +332,11 @@ public void testPollWithPartitionSchemeHashValue() throws Exception { null, KAFKA_TOPIC, KAFKA_VALUE.hashCode() % Integer.parseInt(KAFKA_PARTITIONS), - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); - assertEquals(expected, result.get(0)); + assertRecordsEqual(expected, result.get(0)); } /** @@ -342,7 +368,7 @@ public void testPollWithPartitionSchemeRoundRobin() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); @@ -352,7 +378,7 @@ public void testPollWithPartitionSchemeRoundRobin() throws Exception { null, KAFKA_TOPIC, 1, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); @@ -362,7 +388,7 @@ public void testPollWithPartitionSchemeRoundRobin() throws Exception { null, KAFKA_TOPIC, 2, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); @@ -372,22 +398,22 @@ public void testPollWithPartitionSchemeRoundRobin() throws Exception { null, KAFKA_TOPIC, 0, - Schema.STRING_SCHEMA, + Schema.OPTIONAL_STRING_SCHEMA, null, Schema.BYTES_SCHEMA, KAFKA_VALUE); - assertEquals(expected1, result.get(0)); - assertEquals(expected2, result.get(1)); - assertEquals(expected3, result.get(2)); - assertEquals(expected4, result.get(3)); + assertRecordsEqual(expected1, result.get(0)); + assertRecordsEqual(expected2, result.get(1)); + assertRecordsEqual(expected3, result.get(2)); + assertRecordsEqual(expected4, result.get(3)); } - @Test(expected = RuntimeException.class) + @Test public void testPollExceptionCase() throws Exception { task.start(props); // Could also throw ExecutionException if we wanted to... when(subscriber.pull(any(PullRequest.class)).get()).thenThrow(new InterruptedException()); - task.poll(); + assertEquals(0, task.poll().size()); } private ReceivedMessage createReceivedMessage( diff --git a/load-test-framework/README.md b/load-test-framework/README.md index 2b9a7f40..feec9907 100644 --- a/load-test-framework/README.md +++ b/load-test-framework/README.md @@ -33,11 +33,13 @@ The `--client_types` parameter to sets the clients to test. This is a comma deli to your local machine. 3. Go to the "IAM" tab, find the service account you just created and click on - the dropdown menu named "Role(s)". Under the "Pub/Sub" submenu, select - "Pub/Sub Admin". + the dropdown menu named "Role(s)". Select "Project Editor". The load test + framework requires permissive access since it creates GCE templates, + creates Pub/Sub topics, and modifies firewalls. If you are uncomfortable + with the permissions, please use a new GCP project for running load tests. If you don't see the service account in the list, add a new permission, use - the service account as the member name, and select "Pub/Sub Admin" from the + the service account as the member name, and select "Project Editor" from the role dropdown menu in the window. Now, the service account you just created should appear in the members list diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 21047f54..57503691 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -8,16 +8,6 @@ CompatibilityTesting http://maven.apache.org - - com.google.pubsub - cloud-pubsub-client - 0.2-EXPERIMENTAL - - - com.google.pubsub - pubsub-mapped-api - 1.0-SNAPSHOT - junit junit @@ -33,6 +23,11 @@ jcommander 1.48 + + com.google.cloud + google-cloud-pubsub + 0.24.0-beta + commons-io commons-io @@ -43,36 +38,10 @@ commons-lang3 3.4 - - io.grpc - grpc-all - 1.0.1 - - - io.grpc - grpc-netty - 1.0.1 - - - io.grpc - grpc-protobuf - 1.0.1 - - - io.grpc - grpc-stub - 1.0.1 - - - io.netty - netty-tcnative-boringssl-static - 1.1.33.Fork14 - linux-x86_64 - com.google.guava guava - 20.0 + 22.0 com.google.apis @@ -130,11 +99,6 @@ httpclient 4.2.5 - - com.google.cloud - google-cloud-pubsub - 0.9.3-alpha - com.google.apis google-api-services-sheets @@ -155,11 +119,6 @@ zkclient 0.7 - - org.apache.kafka - kafka-clients - 0.10.0.0 - diff --git a/load-test-framework/python_src/clients/cps_publisher_task.py b/load-test-framework/python_src/clients/cps_publisher_task.py index 035026d5..49b20f57 100644 --- a/load-test-framework/python_src/clients/cps_publisher_task.py +++ b/load-test-framework/python_src/clients/cps_publisher_task.py @@ -22,8 +22,8 @@ from concurrent import futures import grpc import loadtest_pb2 -from google.cloud import pubsub - +from google.cloud.pubsub_v1 import publisher +from google.cloud.pubsub_v1.types import BatchSettings class LoadtestWorkerServicer(loadtest_pb2.LoadtestWorkerServicer): """Provides methods that implement functionality of load test server.""" @@ -34,34 +34,44 @@ def __init__(self): self.batch_size = None self.batch = None self.num_msgs_published = 0 - self.id = int(random.random() * sys.maxint) + self.id = str(int(random.random() * (2 ** 32 - 1))) + self.latencies = [] def Start(self, request, context): self.message_size = request.message_size - self.batch_size = request.pubsub_options.publish_batch_size - self.batch = pubsub.Client().topic(request.topic).batch() + self.batch_size = request.publish_batch_size + self.topic = "projects/" + request.project + "/topics/" + request.topic + self.client = publisher.Client(batch_settings=BatchSettings(max_latency=float(request.publish_batch_duration.seconds) + float(request.publish_batch_duration.nanos) / 1000000000.0)) return loadtest_pb2.StartResponse() + def OnPublishDone(self, start, future): + if future.exception() is not None: + return + end = time.time() + self.lock.acquire() + self.latencies.append(int((end - start) * 1000)) + self.lock.release() + def Execute(self, request, context): self.lock.acquire() sequence_number = self.num_msgs_published self.num_msgs_published += self.batch_size + latencies = self.latencies + self.latencies = [] self.lock.release() - start = time.clock() + start = time.time() for i in range(0, self.batch_size): - self.batch.publish(("A" * self.message_size).encode(), - sendTime=str(int(start * 1000)), - clientId=str(self.id), - sequenceNumber=str(int(sequence_number + i))) - self.batch.commit() - end = time.clock() + self.client.publish(self.topic, ("A" * self.message_size).encode(), + sendTime=str(int(time.time() * 1000)), + clientId=self.id, + sequenceNumber=str(int(sequence_number + i))).add_done_callback(lambda f: self.OnPublishDone(start, f)) response = loadtest_pb2.ExecuteResponse() - response.latencies.extend([int((end - start) * 1000)] * self.batch_size) + response.latencies.extend(latencies) return response if __name__ == "__main__": - port = 6000 + port = "6000" for arg in sys.argv: if arg.startswith("--worker_port="): port = arg.split("=")[1] diff --git a/load-test-framework/python_src/clients/cps_subscriber_task.py b/load-test-framework/python_src/clients/cps_subscriber_task.py new file mode 100644 index 00000000..b5e2fb7e --- /dev/null +++ b/load-test-framework/python_src/clients/cps_subscriber_task.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import threading +import time + +from concurrent import futures +import grpc +import loadtest_pb2 +from google.cloud.pubsub_v1 import subscriber + +class LoadtestWorkerServicer(loadtest_pb2.LoadtestWorkerServicer): + """Provides methods that implement functionality of load test server.""" + + def __init__(self): + self.lock = threading.Lock() + self.latencies = [] + self.recv_msgs = [] + + def ProcessMessage(self, message): + latency = int(time.time() * 1000 - int(message.attributes()["sendTime"])) + identifier = loadtest_pb2.MessageIdentifier() + identifier.publisher_client_id = int(message.attributes()["clientId"]) + identifier.sequence_number = int(message.attributes()["sequenceNumber"]) + message.ack() + self.lock.acquire() + self.latencies.append(latency) + self.recv_msgs.append(identifier) + self.lock.release() + + def Start(self, request, context): + self.message_size = request.message_size + self.batch_size = request.publish_batch_size + subscription = "projects/" + request.project + "/subscriptions/" + request.pubsub_options.subscription + self.client = subscriber.Client() + self.client.subscribe(subscription, lambda msg: self.ProcessMessage(msg)) + return loadtest_pb2.StartResponse() + + def Execute(self, request, context): + response = loadtest_pb2.ExecuteResponse() + self.lock.acquire() + response.latencies.extend(self.latencies) + response.received_messages.extend(self.recv_msgs) + self.latencies = [] + self.recv_msgs = [] + self.lock.release() + return response + + +if __name__ == "__main__": + port = "6000" + for arg in sys.argv: + if arg.startswith("--worker_port="): + port = arg.split("=")[1] + server = grpc.server(futures.ThreadPoolExecutor(max_workers=1000)) + loadtest_pb2.add_LoadtestWorkerServicer_to_server(LoadtestWorkerServicer(), server) + server.add_insecure_port('localhost:' + port) + server.start() + while True: + time.sleep(3600) diff --git a/load-test-framework/python_src/clients/loadtest_pb2.py b/load-test-framework/python_src/clients/loadtest_pb2.py index bfb203c9..95f55926 100644 --- a/load-test-framework/python_src/clients/loadtest_pb2.py +++ b/load-test-framework/python_src/clients/loadtest_pb2.py @@ -21,7 +21,7 @@ name='loadtest.proto', package='google.pubsub.loadtest', syntax='proto3', - serialized_pb=_b('\n\x0eloadtest.proto\x12\x16google.pubsub.loadtest\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xb8\x03\n\x0cStartRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x14\n\x0crequest_rate\x18\x03 \x01(\x05\x12\x14\n\x0cmessage_size\x18\x04 \x01(\x05\x12 \n\x18max_outstanding_requests\x18\x05 \x01(\x05\x12.\n\nstart_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x32\n\rtest_duration\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12\x1c\n\x12number_of_messages\x18\x08 \x01(\x05H\x00\x12?\n\x0epubsub_options\x18\t \x01(\x0b\x32%.google.pubsub.loadtest.PubsubOptionsH\x01\x12=\n\rkafka_options\x18\n \x01(\x0b\x32$.google.pubsub.loadtest.KafkaOptionsH\x01\x12\x1a\n\x12publish_batch_size\x18\x0b \x01(\x05\x42\x11\n\x0fstop_conditionsB\t\n\x07options\"\x0f\n\rStartResponse\"D\n\rPubsubOptions\x12\x14\n\x0csubscription\x18\x01 \x01(\t\x12\x1d\n\x15max_messages_per_pull\x18\x02 \x01(\x05\"3\n\x0cKafkaOptions\x12\x0e\n\x06\x62roker\x18\x01 \x01(\t\x12\x13\n\x0bpoll_length\x18\x02 \x01(\x05\"\x0e\n\x0c\x43heckRequest\"I\n\x11MessageIdentifier\x12\x1b\n\x13publisher_client_id\x18\x01 \x01(\x03\x12\x17\n\x0fsequence_number\x18\x02 \x01(\x05\"\xb6\x01\n\rCheckResponse\x12\x15\n\rbucket_values\x18\x01 \x03(\x03\x12\x33\n\x10running_duration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bis_finished\x18\x03 \x01(\x08\x12\x44\n\x11received_messages\x18\x04 \x03(\x0b\x32).google.pubsub.loadtest.MessageIdentifier\"\x10\n\x0e\x45xecuteRequest\"j\n\x0f\x45xecuteResponse\x12\x11\n\tlatencies\x18\x01 \x03(\x03\x12\x44\n\x11received_messages\x18\x02 \x03(\x0b\x32).google.pubsub.loadtest.MessageIdentifier2\xb6\x01\n\x08Loadtest\x12T\n\x05Start\x12$.google.pubsub.loadtest.StartRequest\x1a%.google.pubsub.loadtest.StartResponse\x12T\n\x05\x43heck\x12$.google.pubsub.loadtest.CheckRequest\x1a%.google.pubsub.loadtest.CheckResponse2\xc2\x01\n\x0eLoadtestWorker\x12T\n\x05Start\x12$.google.pubsub.loadtest.StartRequest\x1a%.google.pubsub.loadtest.StartResponse\x12Z\n\x07\x45xecute\x12&.google.pubsub.loadtest.ExecuteRequest\x1a\'.google.pubsub.loadtest.ExecuteResponseB.\n\x1d\x63om.google.pubsub.flic.commonB\rLoadtestProtob\x06proto3') + serialized_pb=_b('\n\x0eloadtest.proto\x12\x16google.pubsub.loadtest\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa8\x04\n\x0cStartRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x14\n\x0crequest_rate\x18\x03 \x01(\x05\x12\x14\n\x0cmessage_size\x18\x04 \x01(\x05\x12 \n\x18max_outstanding_requests\x18\x05 \x01(\x05\x12.\n\nstart_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x10\x62urn_in_duration\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1a\n\x12publish_batch_size\x18\x0b \x01(\x05\x12\x39\n\x16publish_batch_duration\x18\r \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\rtest_duration\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12\x1c\n\x12number_of_messages\x18\x08 \x01(\x05H\x00\x12?\n\x0epubsub_options\x18\t \x01(\x0b\x32%.google.pubsub.loadtest.PubsubOptionsH\x01\x12=\n\rkafka_options\x18\n \x01(\x0b\x32$.google.pubsub.loadtest.KafkaOptionsH\x01\x42\x11\n\x0fstop_conditionsB\t\n\x07options\"\x0f\n\rStartResponse\"D\n\rPubsubOptions\x12\x14\n\x0csubscription\x18\x01 \x01(\t\x12\x1d\n\x15max_messages_per_pull\x18\x02 \x01(\x05\"\x9e\x01\n\x0cKafkaOptions\x12\x0e\n\x06\x62roker\x18\x01 \x01(\t\x12\x30\n\rpoll_duration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1c\n\x14zookeeper_ip_address\x18\x03 \x01(\t\x12\x1a\n\x12replication_factor\x18\x04 \x01(\x05\x12\x12\n\npartitions\x18\x05 \x01(\x05\"I\n\x11MessageIdentifier\x12\x1b\n\x13publisher_client_id\x18\x01 \x01(\x03\x12\x17\n\x0fsequence_number\x18\x02 \x01(\x05\"M\n\x0c\x43heckRequest\x12=\n\nduplicates\x18\x01 \x03(\x0b\x32).google.pubsub.loadtest.MessageIdentifier\"\xb6\x01\n\rCheckResponse\x12\x15\n\rbucket_values\x18\x01 \x03(\x03\x12\x33\n\x10running_duration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bis_finished\x18\x03 \x01(\x08\x12\x44\n\x11received_messages\x18\x04 \x03(\x0b\x32).google.pubsub.loadtest.MessageIdentifier\"\x10\n\x0e\x45xecuteRequest\"j\n\x0f\x45xecuteResponse\x12\x11\n\tlatencies\x18\x01 \x03(\x03\x12\x44\n\x11received_messages\x18\x02 \x03(\x0b\x32).google.pubsub.loadtest.MessageIdentifier2\xb6\x01\n\x08Loadtest\x12T\n\x05Start\x12$.google.pubsub.loadtest.StartRequest\x1a%.google.pubsub.loadtest.StartResponse\x12T\n\x05\x43heck\x12$.google.pubsub.loadtest.CheckRequest\x1a%.google.pubsub.loadtest.CheckResponse2\xc2\x01\n\x0eLoadtestWorker\x12T\n\x05Start\x12$.google.pubsub.loadtest.StartRequest\x1a%.google.pubsub.loadtest.StartResponse\x12Z\n\x07\x45xecute\x12&.google.pubsub.loadtest.ExecuteRequest\x1a\'.google.pubsub.loadtest.ExecuteResponseB.\n\x1d\x63om.google.pubsub.flic.commonB\rLoadtestProtob\x06proto3') , dependencies=[google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -79,40 +79,54 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='test_duration', full_name='google.pubsub.loadtest.StartRequest.test_duration', index=6, - number=7, type=11, cpp_type=10, label=1, + name='burn_in_duration', full_name='google.pubsub.loadtest.StartRequest.burn_in_duration', index=6, + number=12, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='number_of_messages', full_name='google.pubsub.loadtest.StartRequest.number_of_messages', index=7, - number=8, type=5, cpp_type=1, label=1, + name='publish_batch_size', full_name='google.pubsub.loadtest.StartRequest.publish_batch_size', index=7, + number=11, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='pubsub_options', full_name='google.pubsub.loadtest.StartRequest.pubsub_options', index=8, - number=9, type=11, cpp_type=10, label=1, + name='publish_batch_duration', full_name='google.pubsub.loadtest.StartRequest.publish_batch_duration', index=8, + number=13, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='kafka_options', full_name='google.pubsub.loadtest.StartRequest.kafka_options', index=9, - number=10, type=11, cpp_type=10, label=1, + name='test_duration', full_name='google.pubsub.loadtest.StartRequest.test_duration', index=9, + number=7, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='publish_batch_size', full_name='google.pubsub.loadtest.StartRequest.publish_batch_size', index=10, - number=11, type=5, cpp_type=1, label=1, + name='number_of_messages', full_name='google.pubsub.loadtest.StartRequest.number_of_messages', index=10, + number=8, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='pubsub_options', full_name='google.pubsub.loadtest.StartRequest.pubsub_options', index=11, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='kafka_options', full_name='google.pubsub.loadtest.StartRequest.kafka_options', index=12, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -132,7 +146,7 @@ index=1, containing_type=None, fields=[]), ], serialized_start=108, - serialized_end=548, + serialized_end=660, ) @@ -155,8 +169,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=550, - serialized_end=565, + serialized_start=662, + serialized_end=677, ) @@ -193,8 +207,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=567, - serialized_end=635, + serialized_start=679, + serialized_end=747, ) @@ -213,8 +227,29 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='poll_length', full_name='google.pubsub.loadtest.KafkaOptions.poll_length', index=1, - number=2, type=5, cpp_type=1, label=1, + name='poll_duration', full_name='google.pubsub.loadtest.KafkaOptions.poll_duration', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='zookeeper_ip_address', full_name='google.pubsub.loadtest.KafkaOptions.zookeeper_ip_address', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='replication_factor', full_name='google.pubsub.loadtest.KafkaOptions.replication_factor', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='partitions', full_name='google.pubsub.loadtest.KafkaOptions.partitions', index=4, + number=5, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -231,18 +266,32 @@ extension_ranges=[], oneofs=[ ], - serialized_start=637, - serialized_end=688, + serialized_start=750, + serialized_end=908, ) -_CHECKREQUEST = _descriptor.Descriptor( - name='CheckRequest', - full_name='google.pubsub.loadtest.CheckRequest', +_MESSAGEIDENTIFIER = _descriptor.Descriptor( + name='MessageIdentifier', + full_name='google.pubsub.loadtest.MessageIdentifier', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ + _descriptor.FieldDescriptor( + name='publisher_client_id', full_name='google.pubsub.loadtest.MessageIdentifier.publisher_client_id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sequence_number', full_name='google.pubsub.loadtest.MessageIdentifier.sequence_number', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -255,29 +304,22 @@ extension_ranges=[], oneofs=[ ], - serialized_start=690, - serialized_end=704, + serialized_start=910, + serialized_end=983, ) -_MESSAGEIDENTIFIER = _descriptor.Descriptor( - name='MessageIdentifier', - full_name='google.pubsub.loadtest.MessageIdentifier', +_CHECKREQUEST = _descriptor.Descriptor( + name='CheckRequest', + full_name='google.pubsub.loadtest.CheckRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='publisher_client_id', full_name='google.pubsub.loadtest.MessageIdentifier.publisher_client_id', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sequence_number', full_name='google.pubsub.loadtest.MessageIdentifier.sequence_number', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, + name='duplicates', full_name='google.pubsub.loadtest.CheckRequest.duplicates', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -293,8 +335,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=706, - serialized_end=779, + serialized_start=985, + serialized_end=1062, ) @@ -345,8 +387,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=782, - serialized_end=964, + serialized_start=1065, + serialized_end=1247, ) @@ -369,8 +411,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=966, - serialized_end=982, + serialized_start=1249, + serialized_end=1265, ) @@ -407,11 +449,13 @@ extension_ranges=[], oneofs=[ ], - serialized_start=984, - serialized_end=1090, + serialized_start=1267, + serialized_end=1373, ) _STARTREQUEST.fields_by_name['start_time'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_STARTREQUEST.fields_by_name['burn_in_duration'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION +_STARTREQUEST.fields_by_name['publish_batch_duration'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _STARTREQUEST.fields_by_name['test_duration'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _STARTREQUEST.fields_by_name['pubsub_options'].message_type = _PUBSUBOPTIONS _STARTREQUEST.fields_by_name['kafka_options'].message_type = _KAFKAOPTIONS @@ -427,6 +471,8 @@ _STARTREQUEST.oneofs_by_name['options'].fields.append( _STARTREQUEST.fields_by_name['kafka_options']) _STARTREQUEST.fields_by_name['kafka_options'].containing_oneof = _STARTREQUEST.oneofs_by_name['options'] +_KAFKAOPTIONS.fields_by_name['poll_duration'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION +_CHECKREQUEST.fields_by_name['duplicates'].message_type = _MESSAGEIDENTIFIER _CHECKRESPONSE.fields_by_name['running_duration'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _CHECKRESPONSE.fields_by_name['received_messages'].message_type = _MESSAGEIDENTIFIER _EXECUTERESPONSE.fields_by_name['received_messages'].message_type = _MESSAGEIDENTIFIER @@ -434,8 +480,8 @@ DESCRIPTOR.message_types_by_name['StartResponse'] = _STARTRESPONSE DESCRIPTOR.message_types_by_name['PubsubOptions'] = _PUBSUBOPTIONS DESCRIPTOR.message_types_by_name['KafkaOptions'] = _KAFKAOPTIONS -DESCRIPTOR.message_types_by_name['CheckRequest'] = _CHECKREQUEST DESCRIPTOR.message_types_by_name['MessageIdentifier'] = _MESSAGEIDENTIFIER +DESCRIPTOR.message_types_by_name['CheckRequest'] = _CHECKREQUEST DESCRIPTOR.message_types_by_name['CheckResponse'] = _CHECKRESPONSE DESCRIPTOR.message_types_by_name['ExecuteRequest'] = _EXECUTEREQUEST DESCRIPTOR.message_types_by_name['ExecuteResponse'] = _EXECUTERESPONSE @@ -468,13 +514,6 @@ )) _sym_db.RegisterMessage(KafkaOptions) -CheckRequest = _reflection.GeneratedProtocolMessageType('CheckRequest', (_message.Message,), dict( - DESCRIPTOR = _CHECKREQUEST, - __module__ = 'loadtest_pb2' - # @@protoc_insertion_point(class_scope:google.pubsub.loadtest.CheckRequest) - )) -_sym_db.RegisterMessage(CheckRequest) - MessageIdentifier = _reflection.GeneratedProtocolMessageType('MessageIdentifier', (_message.Message,), dict( DESCRIPTOR = _MESSAGEIDENTIFIER, __module__ = 'loadtest_pb2' @@ -482,6 +521,13 @@ )) _sym_db.RegisterMessage(MessageIdentifier) +CheckRequest = _reflection.GeneratedProtocolMessageType('CheckRequest', (_message.Message,), dict( + DESCRIPTOR = _CHECKREQUEST, + __module__ = 'loadtest_pb2' + # @@protoc_insertion_point(class_scope:google.pubsub.loadtest.CheckRequest) + )) +_sym_db.RegisterMessage(CheckRequest) + CheckResponse = _reflection.GeneratedProtocolMessageType('CheckResponse', (_message.Message,), dict( DESCRIPTOR = _CHECKRESPONSE, __module__ = 'loadtest_pb2' diff --git a/load-test-framework/python_src/clients/requirements.txt b/load-test-framework/python_src/clients/requirements.txt index 8459c193..0a1bbf4a 100644 --- a/load-test-framework/python_src/clients/requirements.txt +++ b/load-test-framework/python_src/clients/requirements.txt @@ -1,2 +1 @@ grpcio -google-cloud-pubsub diff --git a/load-test-framework/ruby_src/Gemfile b/load-test-framework/ruby_src/Gemfile new file mode 100644 index 00000000..83647c92 --- /dev/null +++ b/load-test-framework/ruby_src/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gem "google-cloud-pubsub", github: "GoogleCloudPlatform/google-cloud-ruby", branch: "pubsub" diff --git a/load-test-framework/ruby_src/Gemfile.lock b/load-test-framework/ruby_src/Gemfile.lock new file mode 100644 index 00000000..28cf8e5f --- /dev/null +++ b/load-test-framework/ruby_src/Gemfile.lock @@ -0,0 +1,75 @@ +GIT + remote: git://github.com/GoogleCloudPlatform/google-cloud-ruby.git + revision: 85dd2b3cd718046cbecb727d054ba66fd6523d13 + branch: pubsub + specs: + google-cloud-core (1.0.0) + google-cloud-env (~> 1.0) + googleauth (~> 0.5.1) + google-cloud-env (1.0.1) + faraday (~> 0.11) + google-cloud-pubsub (0.26.0) + concurrent-ruby (~> 1.0) + google-cloud-core (~> 1.0) + google-gax (~> 0.8.0) + grpc (~> 1.1) + grpc-google-iam-v1 (~> 0.6.8) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.1) + public_suffix (~> 2.0, >= 2.0.2) + concurrent-ruby (1.0.5) + faraday (0.12.2) + multipart-post (>= 1.2, < 3) + google-gax (0.8.5) + google-protobuf (~> 3.2) + googleapis-common-protos (~> 1.3.5) + googleauth (~> 0.5.1) + grpc (~> 1.0) + rly (~> 0.2.3) + google-protobuf (3.3.0) + googleapis-common-protos (1.3.5) + google-protobuf (~> 3.2) + grpc (~> 1.0) + googleauth (0.5.3) + faraday (~> 0.12) + jwt (~> 1.4) + logging (~> 2.0) + memoist (~> 0.12) + multi_json (~> 1.11) + os (~> 0.9) + signet (~> 0.7) + grpc (1.4.1) + google-protobuf (~> 3.1) + googleauth (~> 0.5.1) + grpc-google-iam-v1 (0.6.8) + googleapis-common-protos (~> 1.3.1) + googleauth (~> 0.5.1) + grpc (~> 1.0) + jwt (1.5.6) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + multi_json (1.12.1) + multipart-post (2.0.0) + os (0.9.6) + public_suffix (2.0.5) + rly (0.2.3) + signet (0.7.3) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (~> 1.5) + multi_json (~> 1.10) + +PLATFORMS + ruby + +DEPENDENCIES + google-cloud-pubsub! + +BUNDLED WITH + 1.15.3 diff --git a/load-test-framework/ruby_src/cps_publisher_task.rb b/load-test-framework/ruby_src/cps_publisher_task.rb new file mode 100644 index 00000000..f3aed0c6 --- /dev/null +++ b/load-test-framework/ruby_src/cps_publisher_task.rb @@ -0,0 +1,62 @@ +#!/usr/bin/ruby + +require 'grpc' +require 'thread' +require 'google/cloud/pubsub' +require './loadtest_services_pb' + +include Google::Pubsub::Loadtest + +class ServerImpl < Google::Pubsub::Loadtest::LoadtestWorker::Service + + def start(request, _call) + puts "Received Start" + @message_size = request.message_size + @batch_size = request.publish_batch_size + pubsub = Google::Cloud::Pubsub.new(project: request.project) + @publisher = Google::Cloud::Pubsub::AsyncPublisher.new request.topic, pubsub.service, interval: (request.publish_batch_duration.seconds.to_f + request.publish_batch_duration.nanos.to_f / 1000000000.0) + @client_id = (Random.rand(2 ** 31)).to_s + @num_msgs_published = 0 + @latencies = [] + @semaphore = Mutex.new + StartResponse.new + end + + def execute(request, _call) + sequence_number = @num_msgs_published + @num_msgs_published += @batch_size + now = Time.now + latencies = [] + puts "Received execute, going to publish " + @batch_size.to_s + " messages." + 0.upto(@batch_size) do |i| + @publisher.publish ("A" * @message_size), + :sendTime => (now.to_f * 1000).to_i.to_s, + :clientId => @client_id, + :sequenceNumber => (sequence_number + i).to_s do |result| + puts "Message success? " + result.succeeded?.to_s + if result.succeeded? + @semaphore.synchronize do + @latencies.push ((Time.now - now).to_f * 1000).to_i + end + end + end + end + latencies = [] + @semaphore.synchronize do + latencies = @latencies + @latencies = [] + end + puts "Returning " + latencies.to_s + ExecuteResponse.new(latencies: latencies) + end +end + +def main + s = GRPC::RpcServer.new + s.add_http2_port("0.0.0.0:6000", :this_port_is_insecure) + s.handle(ServerImpl.new) + puts "Started server on port 6000." + s.run_till_terminated +end + +main diff --git a/load-test-framework/ruby_src/cps_subscriber_task.rb b/load-test-framework/ruby_src/cps_subscriber_task.rb new file mode 100644 index 00000000..e1b77f27 --- /dev/null +++ b/load-test-framework/ruby_src/cps_subscriber_task.rb @@ -0,0 +1,53 @@ +#!/usr/bin/ruby + +require 'grpc' +require 'thread' +require 'google/cloud/pubsub' +require './loadtest_services_pb' + +include Google::Pubsub::Loadtest + +class ServerImpl < Google::Pubsub::Loadtest::LoadtestWorker::Service + + def start(request, _call) + puts "Received Start" + @latencies = [] + @recv_msgs = [] + @semaphore = Mutex.new + pubsub = Google::Cloud::Pubsub.new(project: request.project) + sub = pubsub.subscription request.pubsub_options.subscription + @subscriber = sub.listen do |msg| + puts "Received message " + msg.to_s + @semaphore.synchronize do + @latencies.push (Time.now.to_f * 1000.0 - msg.attributes['sendTime'].to_f).to_i + @recv_msgs.push MessageIdentifier.new(publisher_client_id: msg.attributes['clientId'].to_i, sequence_number: msg.attributes['sequenceNumber'].to_i) + end + msg.ack! + end + @subscriber.start + StartResponse.new + end + + def execute(request, _call) + latencies = [] + recv_msgs = [] + @semaphore.synchronize do + latencies = @latencies + recv_msgs = @recv_msgs + @latencies = [] + @recv_msgs = [] + end + puts "Returning " + latencies.to_s + ", " + recv_msgs.to_s + ExecuteResponse.new(latencies: latencies, received_messages: recv_msgs) + end +end + +def main + s = GRPC::RpcServer.new + s.add_http2_port("0.0.0.0:6000", :this_port_is_insecure) + s.handle(ServerImpl.new) + puts "Started server on port 6000." + s.run_till_terminated +end + +main diff --git a/load-test-framework/ruby_src/loadtest_pb.rb b/load-test-framework/ruby_src/loadtest_pb.rb new file mode 100644 index 00000000..03407ea0 --- /dev/null +++ b/load-test-framework/ruby_src/loadtest_pb.rb @@ -0,0 +1,76 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: src/main/proto/loadtest.proto + +require 'google/protobuf' + +require 'google/protobuf/duration_pb' +require 'google/protobuf/timestamp_pb' +Google::Protobuf::DescriptorPool.generated_pool.build do + add_message "google.pubsub.loadtest.StartRequest" do + optional :project, :string, 1 + optional :topic, :string, 2 + optional :request_rate, :int32, 3 + optional :message_size, :int32, 4 + optional :max_outstanding_requests, :int32, 5 + optional :start_time, :message, 6, "google.protobuf.Timestamp" + optional :burn_in_duration, :message, 12, "google.protobuf.Duration" + optional :publish_batch_size, :int32, 11 + optional :publish_batch_duration, :message, 13, "google.protobuf.Duration" + oneof :stop_conditions do + optional :test_duration, :message, 7, "google.protobuf.Duration" + optional :number_of_messages, :int32, 8 + end + oneof :options do + optional :pubsub_options, :message, 9, "google.pubsub.loadtest.PubsubOptions" + optional :kafka_options, :message, 10, "google.pubsub.loadtest.KafkaOptions" + end + end + add_message "google.pubsub.loadtest.StartResponse" do + end + add_message "google.pubsub.loadtest.PubsubOptions" do + optional :subscription, :string, 1 + optional :max_messages_per_pull, :int32, 2 + end + add_message "google.pubsub.loadtest.KafkaOptions" do + optional :broker, :string, 1 + optional :poll_duration, :message, 2, "google.protobuf.Duration" + optional :zookeeper_ip_address, :string, 3 + optional :replication_factor, :int32, 4 + optional :partitions, :int32, 5 + end + add_message "google.pubsub.loadtest.MessageIdentifier" do + optional :publisher_client_id, :int64, 1 + optional :sequence_number, :int32, 2 + end + add_message "google.pubsub.loadtest.CheckRequest" do + repeated :duplicates, :message, 1, "google.pubsub.loadtest.MessageIdentifier" + end + add_message "google.pubsub.loadtest.CheckResponse" do + repeated :bucket_values, :int64, 1 + optional :running_duration, :message, 2, "google.protobuf.Duration" + optional :is_finished, :bool, 3 + repeated :received_messages, :message, 4, "google.pubsub.loadtest.MessageIdentifier" + end + add_message "google.pubsub.loadtest.ExecuteRequest" do + end + add_message "google.pubsub.loadtest.ExecuteResponse" do + repeated :latencies, :int64, 1 + repeated :received_messages, :message, 2, "google.pubsub.loadtest.MessageIdentifier" + end +end + +module Google + module Pubsub + module Loadtest + StartRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.StartRequest").msgclass + StartResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.StartResponse").msgclass + PubsubOptions = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.PubsubOptions").msgclass + KafkaOptions = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.KafkaOptions").msgclass + MessageIdentifier = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.MessageIdentifier").msgclass + CheckRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.CheckRequest").msgclass + CheckResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.CheckResponse").msgclass + ExecuteRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.ExecuteRequest").msgclass + ExecuteResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.pubsub.loadtest.ExecuteResponse").msgclass + end + end +end diff --git a/load-test-framework/ruby_src/loadtest_services_pb.rb b/load-test-framework/ruby_src/loadtest_services_pb.rb new file mode 100644 index 00000000..a608b0ae --- /dev/null +++ b/load-test-framework/ruby_src/loadtest_services_pb.rb @@ -0,0 +1,48 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# Source: src/main/proto/loadtest.proto for package 'google.pubsub.loadtest' + +require 'grpc' +require './loadtest_pb' + +module Google + module Pubsub + module Loadtest + module Loadtest + class Service + + include GRPC::GenericService + + self.marshal_class_method = :encode + self.unmarshal_class_method = :decode + self.service_name = 'google.pubsub.loadtest.Loadtest' + + # Starts a load test + rpc :Start, StartRequest, StartResponse + # Checks the status of a load test + rpc :Check, CheckRequest, CheckResponse + end + + Stub = Service.rpc_stub_class + end + module LoadtestWorker + class Service + + include GRPC::GenericService + + self.marshal_class_method = :encode + self.unmarshal_class_method = :decode + self.service_name = 'google.pubsub.loadtest.LoadtestWorker' + + # Starts a worker + rpc :Start, StartRequest, StartResponse + # Executes a command on the worker, returning the latencies of the operations. Since some + # commands consist of multiple operations (i.e. pulls contain many received messages with + # different end to end latencies) a single command can have multiple latencies returned. + rpc :Execute, ExecuteRequest, ExecuteResponse + end + + Stub = Service.rpc_stub_class + end + end + end +end diff --git a/load-test-framework/run.py b/load-test-framework/run.py index 1d0358ae..43030156 100644 --- a/load-test-framework/run.py +++ b/load-test-framework/run.py @@ -32,9 +32,7 @@ def main(project, test, client_types, vms_count, broker): test: The type of test to run. Valid options are 'latency', 'throughput', and 'service'. client_types: The type of clients to run the test against. Valid options - are 'gcloud_java', 'gcloud_python', 'vtk', and 'experimental'. - 'experimental' requires being a part of a whitelist, you can - contact 'cloud-pubsub@google.com' to request access. + are 'gcloud_java', 'gcloud_python', and 'gcloud_ruby' 'vtk'. vms_count: The number of VMs to start for each client type. You must have sufficient Google Compute Engine quota to start vms_count * len(client_types) * 4 cores, and it will take 4 times as much @@ -47,34 +45,40 @@ def main(project, test, client_types, vms_count, broker): subprocess.call(['mvn', 'package']) subprocess.call(['cp', 'target/driver.jar', 'target/classes/gce/']) if not os.path.isfile('./target/classes/gce/cps.zip'): + go_package = 'cloud.google.com/go/pubsub/loadtest/cmd' + go_bin_location = './target/loadtest-go' + return_code = subprocess.call(['go', 'build', '-o', go_bin_location, go_package]) + if return_code != 0: + sys.exit('cannot build Go load tester, maybe run `go get -u {}`?'.format(go_package)) + subprocess.call([ 'zip', './target/classes/gce/cps.zip', './python_src/clients/cps_publisher_task.py', + './python_src/clients/cps_subscriber_task.py', './python_src/clients/loadtest_pb2.py', - './python_src/clients/requirements.txt' + './python_src/clients/requirements.txt', + './ruby_src/Gemfile', + './ruby_src/loadtest_pb.rb', + './ruby_src/loadtest_services_pb.rb', + './ruby_src/cps_publisher_task.rb', + './ruby_src/cps_subscriber_task.rb' , + go_bin_location ]) arg_list = ['java', '-jar', 'target/driver.jar', '--project', project] gcloud_subscriber_count = 0 for client in client_types: if client == 'gcloud_python': arg_list.append('--cps_gcloud_python_publisher_count=' + str(vms_count)) - gcloud_subscriber_count += vms_count + arg_list.append('--cps_gcloud_python_subscriber_count=' + str(vms_count)) + elif client == 'gcloud_ruby': + arg_list.append('--cps_gcloud_ruby_publisher_count=' + str(vms_count)) + arg_list.append('--cps_gcloud_ruby_subscriber_count=' + str(vms_count)) elif client == 'gcloud_java': arg_list.append('--cps_gcloud_java_publisher_count=' + str(vms_count)) - gcloud_subscriber_count += vms_count + arg_list.append('--cps_gcloud_java_subscriber_count=' + str(vms_count)) elif client == 'gcloud_go': arg_list.append('--cps_gcloud_go_publisher_count=' + str(vms_count)) - gcloud_subscriber_count += vms_count - elif client == 'vtk': - arg_list.append('--cps_vtk_java_publisher_count=' + str(vms_count)) - gcloud_subscriber_count += vms_count - elif client == 'experimental': - arg_list.append('--cps_experimental_java_publisher_count=' + str( - vms_count)) - arg_list.append('--cps_experimental_java_subscriber_count=' + str( - vms_count)) - arg_list.append('--cps_gcloud_java_subscriber_count=' + str( - gcloud_subscriber_count)) + arg_list.append('--cps_gcloud_go_subscriber_count=' + str(vms_count)) if broker: arg_list.extend([ '--broker=' + broker, '--kafka_publisher_count=' + str(vms_count), @@ -100,7 +104,7 @@ def main(project, test, client_types, vms_count, broker): '--loadtest_duration=10m', '--burn_in_duration=2m', '--publish_batch_duration=50ms', '--cores=16' ]) - print ' '.join(arg_list) + print(' '.join(arg_list)) subprocess.call(arg_list) @@ -137,11 +141,11 @@ def main(project, test, client_types, vms_count, broker): if len(client_types_arg) == 0: client_types_arg = set(['gcloud_java']) if not client_types_arg.issubset( - set(['gcloud_python', 'gcloud_java', 'gcloud_go', 'vtk', 'experimental'])): + set(['gcloud_python', 'gcloud_java', 'gcloud_go', 'gcloud_ruby'])): sys.exit( 'Invalid --client_type parameter given. Must be a comma deliminated ' 'sequence of client types. Allowed client types are \'gcloud_python\', ' - '\'gcloud_java\', \'vtk\', and \'experimental\'. (' + ','.join( + '\'gcloud_java\', and \'gcloud_ruby\'. (' + ','.join( client_types_arg) + ') was provided. Kafka is assumed if --broker is provided.') main(project_arg, test_arg, client_types_arg, vms_count_arg, broker_arg) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSPublisherTask.java deleted file mode 100644 index 182930fe..00000000 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSPublisherTask.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.experimental; - -import com.beust.jcommander.JCommander; -import com.google.cloud.pubsub.Publisher; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.ByteString; -import com.google.protobuf.util.Durations; -import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler; -import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.common.Task.RunResult; -import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.v1.PubsubMessage; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Runs a task that publishes messages to a Cloud Pub/Sub topic. */ -class CPSPublisherTask extends Task { - private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); - private Publisher publisher; - private final int batchSize; - private final ByteString payload; - private final Integer id; - private final AtomicInteger sequenceNumber = new AtomicInteger(0); - - private CPSPublisherTask(StartRequest request) { - super(request, "experimental", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); - this.batchSize = request.getPublishBatchSize(); - this.id = (new Random()).nextInt(); - try { - this.publisher = - Publisher.Builder.newBuilder( - "projects/" + request.getProject() + "/topics/" + request.getTopic()) - .setMaxBatchDuration( - Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) - .setMaxBatchBytes(9500000) - .setMaxBatchMessages(950) - .setMaxOutstandingBytes(1000000000) // 1 GB - .build(); - } catch (Exception e) { - throw new RuntimeException(e); - } - this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); - } - - public static void main(String[] args) throws Exception { - LoadTestRunner.Options options = new LoadTestRunner.Options(); - new JCommander(options, args); - LoadTestRunner.run(options, CPSPublisherTask::new); - } - - @Override - public ListenableFuture doRun() { - try { - List> results = new ArrayList<>(); - String sendTime = String.valueOf(System.currentTimeMillis()); - for (int i = 0; i < batchSize; i++) { - results.add( - publisher.publish( - PubsubMessage.newBuilder() - .setData(payload) - .putAttributes("sendTime", sendTime) - .putAttributes("clientId", id.toString()) - .putAttributes( - "sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) - .build())); - } - return Futures.transform( - Futures.allAsList(results), response -> RunResult.fromBatchSize(batchSize)); - } catch (Throwable t) { - log.error("Flow control error.", t); - return Futures.immediateFailedFuture(t); - } - } - - @Override - protected void shutdown() { - publisher.shutdown(); - } -} diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSSubscriberTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSSubscriberTask.java deleted file mode 100644 index 8c174329..00000000 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/experimental/CPSSubscriberTask.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.experimental; - -import com.beust.jcommander.JCommander; -import com.google.cloud.pubsub.Subscriber; -import com.google.cloud.pubsub.Subscriber.MessageReceiver.AckReply; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler; -import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.common.Task.RunResult; -import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.v1.PubsubMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Runs a task that subscribes to a Cloud Pub/Sub topic. */ -class CPSSubscriberTask extends Task implements Subscriber.MessageReceiver { - private static final Logger log = LoggerFactory.getLogger(CPSSubscriberTask.class); - private final Subscriber subscriber; - - private CPSSubscriberTask(StartRequest request) { - super(request, "experimental", MetricsHandler.MetricName.END_TO_END_LATENCY); - try { - this.subscriber = - Subscriber.Builder.newBuilder( - "projects/" - + request.getProject() - + "/subscriptions/" - + request.getPubsubOptions().getSubscription(), - this) - .build(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static void main(String[] args) throws Exception { - LoadTestRunner.Options options = new LoadTestRunner.Options(); - new JCommander(options, args); - LoadTestRunner.run(options, CPSSubscriberTask::new); - } - - @Override - public ListenableFuture receiveMessage(PubsubMessage message) { - recordMessageLatency( - Integer.parseInt(message.getAttributesMap().get("clientId")), - Integer.parseInt(message.getAttributesMap().get("sequenceNumber")), - System.currentTimeMillis() - Long.parseLong(message.getAttributesMap().get("sendTime"))); - return Futures.immediateFuture(AckReply.ACK); - } - - @Override - public ListenableFuture doRun() { - synchronized (subscriber) { - if (subscriber.isRunning()) { - return Futures.immediateFuture(RunResult.empty()); - } - try { - subscriber.startAsync().awaitRunning(); - } catch (Exception e) { - log.error("Fatal error from subscriber.", e); - return Futures.immediateFailedFuture(e); - } - return Futures.immediateFuture(RunResult.empty()); - } - } - - @Override - public void shutdown() { - synchronized (subscriber) { - subscriber.stopAsync().awaitTerminated(); - } - } -} diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java index 7d5a3378..2dcb5573 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java @@ -16,10 +16,10 @@ package com.google.pubsub.clients.gcloud; import com.beust.jcommander.JCommander; -import com.google.api.gax.core.RpcFutureCallback; -import com.google.api.gax.grpc.BundlingSettings; -import com.google.api.gax.grpc.FlowControlSettings; -import com.google.cloud.pubsub.spi.v1.Publisher; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.gax.batching.BatchingSettings; +import com.google.cloud.pubsub.v1.Publisher; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; @@ -33,10 +33,11 @@ import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.v1.TopicName; import java.util.Random; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; -import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.threeten.bp.Duration; /** * Runs a task that publishes messages to a Cloud Pub/Sub topic. @@ -48,29 +49,28 @@ class CPSPublisherTask extends Task { private final int batchSize; private final Integer id; private final AtomicInteger sequenceNumber = new AtomicInteger(0); + private final Semaphore outstandingBytes = new Semaphore(1000000000); // 1 GB + private final int messageSize; private CPSPublisherTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); try { this.publisher = - Publisher.newBuilder(TopicName.create(request.getProject(), request.getTopic())) - .setBundlingSettings( - BundlingSettings.newBuilder() - .setDelayThreshold( - Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) - .setRequestByteThreshold(9500000L) + Publisher.defaultBuilder(TopicName.create(request.getProject(), request.getTopic())) + .setBatchingSettings( + BatchingSettings.newBuilder() .setElementCountThreshold(950L) + .setRequestByteThreshold(9500000L) + .setDelayThreshold( + Duration.ofMillis(Durations.toMillis(request.getPublishBatchDuration()))) .build()) - .setFlowControlSettings( - FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(1000000000) - .build()) // 1 GB .build(); } catch (Exception e) { throw new RuntimeException(e); } this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); this.batchSize = request.getPublishBatchSize(); + this.messageSize = request.getMessageSize(); this.id = (new Random()).nextInt(); } @@ -82,40 +82,38 @@ public static void main(String[] args) throws Exception { @Override public ListenableFuture doRun() { - try { - AtomicInteger numPending = new AtomicInteger(batchSize); - final SettableFuture done = SettableFuture.create(); - String sendTime = String.valueOf(System.currentTimeMillis()); - for (int i = 0; i < batchSize; i++) { - publisher - .publish( - PubsubMessage.newBuilder() - .setData(payload) - .putAttributes("sendTime", sendTime) - .putAttributes("clientId", id.toString()) - .putAttributes( - "sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) - .build()) - .addCallback( - new RpcFutureCallback() { - @Override - public void onSuccess(String s) { - if (numPending.decrementAndGet() == 0) { - done.set(RunResult.fromBatchSize(batchSize)); - } - } + AtomicInteger numPending = new AtomicInteger(batchSize); + final SettableFuture done = SettableFuture.create(); + String sendTime = String.valueOf(System.currentTimeMillis()); + if (!outstandingBytes.tryAcquire(batchSize * messageSize)) { + return Futures.immediateFailedFuture(new Exception("Flow control limits reached.")); + } + for (int i = 0; i < batchSize; i++) { + ApiFutures.addCallback(publisher + .publish( + PubsubMessage.newBuilder() + .setData(payload) + .putAttributes("sendTime", sendTime) + .putAttributes("clientId", id.toString()) + .putAttributes( + "sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) + .build()), new ApiFutureCallback() { + @Override + public void onSuccess(String messageId) { + outstandingBytes.release(messageSize); + if (numPending.decrementAndGet() == 0) { + done.set(RunResult.fromBatchSize(batchSize)); + } + } - @Override - public void onFailure(Throwable t) { - done.setException(t); - } - }); - } - return done; - } catch (Throwable t) { - log.error("Flow control error.", t); - return Futures.immediateFailedFuture(t); + @Override + public void onFailure(Throwable t) { + outstandingBytes.release(messageSize); + done.setException(t); + } + }); } + return done; } @Override diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java index 8352e77e..ab8be884 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java @@ -16,10 +16,9 @@ package com.google.pubsub.clients.gcloud; import com.beust.jcommander.JCommander; -import com.google.cloud.pubsub.spi.v1.AckReply; -import com.google.cloud.pubsub.spi.v1.AckReplyConsumer; -import com.google.cloud.pubsub.spi.v1.MessageReceiver; -import com.google.cloud.pubsub.spi.v1.Subscriber; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.pubsub.clients.common.LoadTestRunner; @@ -35,16 +34,18 @@ /** Runs a task that consumes messages from a Cloud Pub/Sub subscription. */ class CPSSubscriberTask extends Task implements MessageReceiver { private static final Logger log = LoggerFactory.getLogger(CPSSubscriberTask.class); - private final Subscriber subscriber; + private final SubscriptionName subscription; + private Subscriber subscriber; + private boolean shuttingDown = false; private CPSSubscriberTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.END_TO_END_LATENCY); + this.subscription = + SubscriptionName.create(request.getProject(), request.getPubsubOptions().getSubscription()); try { this.subscriber = - Subscriber.newBuilder( - SubscriptionName.create( - request.getProject(), request.getPubsubOptions().getSubscription()), - this) + Subscriber.defaultBuilder(this.subscription, this) + .setParallelPullCount(Runtime.getRuntime().availableProcessors() * 5) .build(); } catch (Exception e) { throw new RuntimeException(e); @@ -57,7 +58,7 @@ public void receiveMessage(final PubsubMessage message, final AckReplyConsumer c Integer.parseInt(message.getAttributesMap().get("clientId")), Integer.parseInt(message.getAttributesMap().get("sequenceNumber")), System.currentTimeMillis() - Long.parseLong(message.getAttributesMap().get("sendTime"))); - consumer.accept(AckReply.ACK, null); + consumer.ack(); } public static void main(String[] args) throws Exception { @@ -68,14 +69,19 @@ public static void main(String[] args) throws Exception { @Override public ListenableFuture doRun() { - synchronized (subscriber) { + synchronized (this) { if (subscriber.isRunning()) { return Futures.immediateFuture(RunResult.empty()); } + if (shuttingDown) { + return Futures.immediateFailedFuture( + new IllegalStateException("the task is shutting down")); + } try { subscriber.startAsync().awaitRunning(); } catch (Exception e) { log.error("Fatal error from subscriber.", e); + subscriber = Subscriber.defaultBuilder(this.subscription, this).build(); return Futures.immediateFailedFuture(e); } return Futures.immediateFuture(RunResult.empty()); @@ -84,8 +90,16 @@ public ListenableFuture doRun() { @Override public void shutdown() { - synchronized (subscriber) { - subscriber.stopAsync().awaitTerminated(); + Subscriber subscriber; + synchronized (this) { + if (shuttingDown) { + throw new IllegalStateException("the task is already shutting down"); + } + shuttingDown = true; + subscriber = this.subscriber; } + // We must stop out of the lock. Stopping waits for all messages to be processed, + // and processing the messages needs to lock. + subscriber.stopAsync().awaitTerminated(); } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/vtk/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/vtk/CPSPublisherTask.java deleted file mode 100644 index b096c66f..00000000 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/vtk/CPSPublisherTask.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -package com.google.pubsub.clients.vtk; - -import com.beust.jcommander.JCommander; -import com.google.api.gax.core.RpcFutureCallback; -import com.google.cloud.pubsub.spi.v1.PublisherClient; -import com.google.cloud.pubsub.spi.v1.PublisherSettings; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import com.google.protobuf.ByteString; -import com.google.protobuf.util.Durations; -import com.google.pubsub.clients.common.LoadTestRunner; -import com.google.pubsub.clients.common.MetricsHandler; -import com.google.pubsub.clients.common.Task; -import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.v1.PublishRequest; -import com.google.pubsub.v1.PublishResponse; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.TopicName; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Runs a task that publishes messages to a Cloud Pub/Sub topic. */ -class CPSPublisherTask extends Task { - private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); - private final String topic; - private final PublisherClient publisherApi; - private final ByteString payload; - private final int batchSize; - private final Integer id; - private final AtomicInteger sequenceNumber = new AtomicInteger(0); - - private CPSPublisherTask(StartRequest request) { - super(request, "vtk", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); - PublisherSettings.Builder publisherSettingsBuilder = PublisherSettings.defaultBuilder(); - publisherSettingsBuilder - .publishSettings() - .getBundlingSettingsBuilder() - .setDelayThreshold(Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) - .setElementCountThreshold(950L) - .setRequestByteThreshold(9500000L); - try { - this.publisherApi = PublisherClient.create(publisherSettingsBuilder.build()); - } catch (Exception e) { - throw new RuntimeException("Error creating publisher API.", e); - } - this.topic = TopicName.create(request.getProject(), request.getTopic()).toString(); - this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); - this.batchSize = request.getPublishBatchSize(); - this.id = (new Random()).nextInt(); - } - - public static void main(String[] args) throws Exception { - LoadTestRunner.Options options = new LoadTestRunner.Options(); - new JCommander(options, args); - LoadTestRunner.run(options, CPSPublisherTask::new); - } - - @Override - public ListenableFuture doRun() { - List messages = new ArrayList<>(batchSize); - String sendTime = String.valueOf(System.currentTimeMillis()); - for (int i = 0; i < batchSize; i++) { - messages.add( - PubsubMessage.newBuilder() - .setData(payload) - .putAttributes("sendTime", sendTime) - .putAttributes("clientId", id.toString()) - .putAttributes("sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) - .build()); - } - PublishRequest request = - PublishRequest.newBuilder().setTopic(topic).addAllMessages(messages).build(); - SettableFuture result = SettableFuture.create(); - publisherApi - .publishCallable() - .futureCall(request) - .addCallback( - new RpcFutureCallback() { - @Override - public void onSuccess(PublishResponse s) { - result.set(RunResult.fromBatchSize(batchSize)); - } - - @Override - public void onFailure(Throwable t) { - result.setException(t); - } - }); - return result; - } -} diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index fc2aa02d..fbe16b66 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -95,28 +95,28 @@ public class Driver { private int cpsGcloudJavaSubscriberCount = 0; @Parameter( - names = {"--cps_experimental_java_publisher_count"}, + names = {"--cps_gcloud_python_publisher_count"}, description = "Number of CPS publishers of this type to start." ) - private int cpsExperimentalJavaPublisherCount = 0; + private int cpsGcloudPythonPublisherCount = 0; @Parameter( - names = {"--cps_experimental_java_subscriber_count"}, + names = {"--cps_gcloud_python_subscriber_count"}, description = "Number of CPS subscribers of this type to start." ) - private int cpsExperimentalJavaSubscriberCount = 0; + private int cpsGcloudPythonSubscriberCount = 0; @Parameter( - names = {"--cps_vtk_java_publisher_count"}, + names = {"--cps_gcloud_ruby_publisher_count"}, description = "Number of CPS publishers of this type to start." ) - private int cpsVtkJavaPublisherCount = 0; + private int cpsGcloudRubyPublisherCount = 0; @Parameter( - names = {"--cps_gcloud_python_publisher_count"}, - description = "Number of CPS publishers of this type to start." + names = {"--cps_gcloud_ruby_subscriber_count"}, + description = "Number of CPS subscribers of this type to start." ) - private int cpsGcloudPythonPublisherCount = 0; + private int cpsGcloudRubySubscriberCount = 0; @Parameter( names = {"--cps_gcloud_go_publisher_count"}, @@ -124,6 +124,12 @@ public class Driver { ) private int cpsGcloudGoPublisherCount = 0; + @Parameter( + names = {"--cps_gcloud_go_subscriber_count"}, + description = "Number of CPS publishers of this type to start." + ) + private int cpsGcloudGoSubscriberCount = 0; + @Parameter( names = {"--kafka_publisher_count"}, description = "Number of Kafka publishers to start." @@ -136,12 +142,6 @@ public class Driver { ) private int kafkaSubscriberCount = 0; - @Parameter( - names = {"--cps_mapped_java_publisher_count"}, - description = "Number of cps mapped publishers to start." - ) - private int cpsMappedJavaPublisherCount = 0; - @Parameter( names = {"--message_size", "-m"}, description = "Message size in bytes (only when publishing messages).", @@ -171,9 +171,9 @@ public class Driver { private int publishBatchSize = 10; @Parameter( - names = {"--publish_batch_duration"}, - description = "The maximum duration to wait for more messages to batch together.", - converter = DurationConverter.class + names = {"--publish_batch_duration"}, + description = "The maximum duration to wait for more messages to batch together.", + converter = DurationConverter.class ) private Duration publishBatchDuration = Durations.fromMillis(0); @@ -375,25 +375,16 @@ public void run(BiFunction, Controller> contr new ClientParams(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER, null), cpsGcloudPythonPublisherCount); } + if (cpsGcloudRubyPublisherCount > 0) { + clientParamsMap.put( + new ClientParams(ClientType.CPS_GCLOUD_RUBY_PUBLISHER, null), + cpsGcloudRubyPublisherCount); + } if (cpsGcloudGoPublisherCount > 0) { clientParamsMap.put( new ClientParams(ClientType.CPS_GCLOUD_GO_PUBLISHER, null), cpsGcloudGoPublisherCount); } - if (cpsExperimentalJavaPublisherCount > 0) { - clientParamsMap.put( - new ClientParams(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER, null), - cpsExperimentalJavaPublisherCount); - } - if (cpsVtkJavaPublisherCount > 0) { - clientParamsMap.put( - new ClientParams(ClientType.CPS_VTK_JAVA_PUBLISHER, null), cpsVtkJavaPublisherCount); - } - if (cpsMappedJavaPublisherCount > 0) { - clientParamsMap.put( - new ClientParams(ClientType.CPS_MAPPED_JAVA_PUBLISHER, null), - cpsMappedJavaPublisherCount); - } if (kafkaPublisherCount > 0) { clientParamsMap.put( new ClientParams(ClientType.KAFKA_PUBLISHER, null), kafkaPublisherCount); @@ -423,24 +414,32 @@ public void run(BiFunction, Controller> contr // cpsSubscriberCount subscribers cumulatively among each of the subscriptions. for (int i = 0; i < cpsSubscriptionFanout; ++i) { if (cpsGcloudJavaSubscriberCount > 0) { - Preconditions.checkArgument( - cpsGcloudJavaPublisherCount + cpsGcloudPythonPublisherCount + - cpsVtkJavaPublisherCount + cpsGcloudGoPublisherCount + cpsMappedJavaPublisherCount - > 0, - "--cps_gcloud_java_publisher, --cps_gcloud_go_publisher, --cps_gcloud_python_publisher," - + "or --cps_mapped_java_publisher must be > 0."); + Preconditions.checkArgument(cpsGcloudJavaPublisherCount > 0, + "--cps_gcloud_java_publisher must be > 0."); clientParamsMap.put( - new ClientParams(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER, "gcloud-subscription" + i), + new ClientParams(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER, "gcloud-java-subscription" + i), cpsGcloudJavaSubscriberCount / cpsSubscriptionFanout); } - if (cpsExperimentalJavaSubscriberCount > 0) { - Preconditions.checkArgument( - cpsExperimentalJavaPublisherCount > 0, - "--cps_experimental_java_publisher or --cps_vtk_java_publisher must be > 0."); + if (cpsGcloudGoSubscriberCount > 0) { + Preconditions.checkArgument(cpsGcloudGoPublisherCount > 0, + "--cps_gcloud_go_publisher must be > 0."); + clientParamsMap.put( + new ClientParams(ClientType.CPS_GCLOUD_GO_SUBSCRIBER, "gcloud-go-subscription" + i), + cpsGcloudGoSubscriberCount / cpsSubscriptionFanout); + } + if (cpsGcloudPythonSubscriberCount > 0) { + Preconditions.checkArgument(cpsGcloudPythonPublisherCount > 0, + "--cps_gcloud_python_publisher must be > 0."); clientParamsMap.put( - new ClientParams( - ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER, "experimental-subscription" + i), - cpsExperimentalJavaSubscriberCount / cpsSubscriptionFanout); + new ClientParams(ClientType.CPS_GCLOUD_PYTHON_SUBSCRIBER, "gcloud-python-subscription" + i), + cpsGcloudPythonSubscriberCount / cpsSubscriptionFanout); + } + if (cpsGcloudRubySubscriberCount > 0) { + Preconditions.checkArgument(cpsGcloudRubyPublisherCount > 0, + "--cps_gcloud_ruby_publisher must be > 0."); + clientParamsMap.put( + new ClientParams(ClientType.CPS_GCLOUD_RUBY_SUBSCRIBER, "gcloud-ruby-subscription" + i), + cpsGcloudRubySubscriberCount / cpsSubscriptionFanout); } } // Set static variables. @@ -475,14 +474,23 @@ public void run(BiFunction, Controller> contr } if (maxPublishLatencyTest) { controller = controllerFunction.apply(project, clientParamsMap); + if (controller == null) { + System.exit(1); + } runMaxPublishLatencyTest(); } else if (maxSubscriberThroughputTest) { controller = controllerFunction.apply(project, clientParamsMap); + if (controller == null) { + System.exit(1); + } runMaxSubscriberThroughputTest(); } else if (numCoresTest) { runNumCoresTest(clientParamsMap, controllerFunction); } else { controller = controllerFunction.apply(project, clientParamsMap); + if (controller == null) { + System.exit(1); + } Map statsMap = runTest(null); GnuPlot.plot(statsMap); CsvOutput.outputStats(statsMap); @@ -506,9 +514,9 @@ private Map runTest(Runnable whileRunning) new MessageTracker( numberOfMessages, cpsGcloudJavaPublisherCount - + cpsExperimentalJavaPublisherCount - + cpsVtkJavaPublisherCount - + cpsGcloudPythonPublisherCount); + + cpsGcloudPythonPublisherCount + + cpsGcloudRubyPublisherCount + + cpsGcloudGoPublisherCount); controller.startClients(messageTracker); if (whileRunning != null) { whileRunning.run(); @@ -627,6 +635,9 @@ private void runNumCoresTest( CsvOutput csv = new CsvOutput(); for (cores = 1; cores <= 16; cores *= 2) { controller = controllerFunction.apply(project, clientParamsMap); + if (controller == null) { + System.exit(1); + } statsMap = runTest(null); gnuPlot.addCoresResult(cores, statsMap); csv.addCoresResult(cores, statsMap); @@ -768,3 +779,4 @@ public Duration convert(String value) { } } } + diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index de5ec9d8..597ab002 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -29,7 +29,7 @@ import com.google.pubsub.flic.common.LoadtestProto.PubsubOptions; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import com.google.pubsub.flic.common.LoadtestProto.StartResponse; -import io.grpc.netty.NettyChannelBuilder; +import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; @@ -104,21 +104,21 @@ public Client( public static String getTopicSuffix(ClientType clientType) { switch (clientType) { - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: - case CPS_EXPERIMENTAL_JAVA_SUBSCRIBER: - return "experimental"; case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_JAVA_SUBSCRIBER: - case CPS_GCLOUD_PYTHON_PUBLISHER: + return "gcloud-java"; case CPS_GCLOUD_GO_PUBLISHER: - return "gcloud"; - case CPS_VTK_JAVA_PUBLISHER: - return "vtk"; + case CPS_GCLOUD_GO_SUBSCRIBER: + return "gcloud-go"; + case CPS_GCLOUD_PYTHON_PUBLISHER: + case CPS_GCLOUD_PYTHON_SUBSCRIBER: + return "gcloud-python"; + case CPS_GCLOUD_RUBY_PUBLISHER: + case CPS_GCLOUD_RUBY_SUBSCRIBER: + return "gcloud-ruby"; case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; - case CPS_MAPPED_JAVA_PUBLISHER: - return "mapped"; } return null; } @@ -136,9 +136,9 @@ private LoadtestGrpc.LoadtestStub getStub() { return stubFactory.get(); } return LoadtestGrpc.newStub( - NettyChannelBuilder.forAddress(networkAddress, PORT) + ManagedChannelBuilder.forAddress(networkAddress, PORT) .usePlaintext(true) - .maxMessageSize(100000000) + .maxInboundMessageSize(100000000) .build()); } @@ -171,14 +171,11 @@ void start(MessageTracker messageTracker) throws Throwable { requestBuilder.setTestDuration(loadtestDuration); } switch (clientType) { - case CPS_EXPERIMENTAL_JAVA_SUBSCRIBER: - requestBuilder.setPubsubOptions(PubsubOptions.newBuilder() - .setSubscription(subscription)); - break; case CPS_GCLOUD_JAVA_SUBSCRIBER: - requestBuilder.setPubsubOptions(PubsubOptions.newBuilder() - .setSubscription(subscription) - .setMaxMessagesPerPull(maxMessagesPerPull)); + case CPS_GCLOUD_GO_SUBSCRIBER: + case CPS_GCLOUD_PYTHON_SUBSCRIBER: + case CPS_GCLOUD_RUBY_SUBSCRIBER: + requestBuilder.setPubsubOptions(PubsubOptions.newBuilder().setSubscription(subscription)); break; case KAFKA_PUBLISHER: requestBuilder.setKafkaOptions(KafkaOptions.newBuilder() @@ -195,12 +192,10 @@ void start(MessageTracker messageTracker) throws Throwable { .setReplicationFactor(replicationFactor) .setPartitions(partitions)); break; - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: + case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: - case CPS_VTK_JAVA_PUBLISHER: - case CPS_MAPPED_JAVA_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -299,25 +294,23 @@ public void onCompleted() { * An enum representing the possible client types. */ public enum ClientType { - CPS_EXPERIMENTAL_JAVA_PUBLISHER, - CPS_EXPERIMENTAL_JAVA_SUBSCRIBER, CPS_GCLOUD_JAVA_PUBLISHER, CPS_GCLOUD_JAVA_SUBSCRIBER, CPS_GCLOUD_PYTHON_PUBLISHER, + CPS_GCLOUD_PYTHON_SUBSCRIBER, + CPS_GCLOUD_RUBY_PUBLISHER, + CPS_GCLOUD_RUBY_SUBSCRIBER, CPS_GCLOUD_GO_PUBLISHER, - CPS_VTK_JAVA_PUBLISHER, + CPS_GCLOUD_GO_SUBSCRIBER, KAFKA_PUBLISHER, - KAFKA_SUBSCRIBER, - CPS_MAPPED_JAVA_PUBLISHER; + KAFKA_SUBSCRIBER; public boolean isCpsPublisher() { switch (this) { - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: + case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: - case CPS_VTK_JAVA_PUBLISHER: - case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -335,13 +328,11 @@ public boolean isKafkaPublisher() { public boolean isPublisher() { switch (this) { - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: + case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: - case CPS_VTK_JAVA_PUBLISHER: case KAFKA_PUBLISHER: - case CPS_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -350,16 +341,16 @@ public boolean isPublisher() { public ClientType getSubscriberType() { switch (this) { - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: - return CPS_EXPERIMENTAL_JAVA_SUBSCRIBER; case CPS_GCLOUD_JAVA_PUBLISHER: - case CPS_GCLOUD_PYTHON_PUBLISHER: - case CPS_GCLOUD_GO_PUBLISHER: - case CPS_VTK_JAVA_PUBLISHER: - case CPS_MAPPED_JAVA_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; + case CPS_GCLOUD_GO_PUBLISHER: + return CPS_GCLOUD_GO_SUBSCRIBER; + case CPS_GCLOUD_PYTHON_PUBLISHER: + return CPS_GCLOUD_PYTHON_SUBSCRIBER; + case CPS_GCLOUD_RUBY_PUBLISHER: + return CPS_GCLOUD_RUBY_SUBSCRIBER; default: return this; } diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java index 2af215d1..3654640d 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java @@ -324,6 +324,7 @@ public static GCEController newGCEController( .setApplicationName("Cloud Pub/Sub Loadtest Framework") .build()); } catch (Throwable t) { + log.error("Unable to initialize GCE: ", t); return null; } } @@ -533,6 +534,9 @@ private void addInstanceGroupInfo(String zone, ClientParams params) throws IOExc * startup script used. */ private InstanceTemplate defaultInstanceTemplate(String type) { + AccessConfig config = new AccessConfig(); + config.setType("ONE_TO_ONE_NAT"); + config.setName("External NAT"); return new InstanceTemplate() .setName("cps-loadtest-" + type + "-" + cores) .setProperties(new InstanceProperties() @@ -544,7 +548,7 @@ private InstanceTemplate defaultInstanceTemplate(String type) { .setSourceImage(SOURCE_FAMILY)))) .setNetworkInterfaces(Collections.singletonList(new NetworkInterface() .setNetwork("global/networks/default") - .setAccessConfigs(Collections.singletonList(new AccessConfig())))) + .setAccessConfigs(Collections.singletonList(config)))) .setMetadata(new Metadata() .setItems(ImmutableList.of( new Metadata.Items() diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 6a3c5c44..0f5c2a99 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -74,16 +74,17 @@ private void fillClientCounts(Map> types) { Map countMap = paramsMap.keySet().stream(). collect(Collectors.groupingBy( ClientParams::getClientType, Collectors.summingInt(t -> 1))); - cpsPublisherCount += (countMap.get(ClientType.CPS_GCLOUD_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_GCLOUD_JAVA_PUBLISHER) : 0; - cpsPublisherCount += (countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) != null) ? countMap.get(ClientType.CPS_GCLOUD_PYTHON_PUBLISHER) : 0; - cpsPublisherCount += (countMap.get(ClientType.CPS_GCLOUD_GO_PUBLISHER) != null) ? countMap.get(ClientType.CPS_GCLOUD_GO_PUBLISHER) : 0; - cpsPublisherCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_PUBLISHER): 0; - cpsPublisherCount += (countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_VTK_JAVA_PUBLISHER) : 0; - cpsPublisherCount += (countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) != null) ? countMap.get(ClientType.CPS_MAPPED_JAVA_PUBLISHER) : 0; - cpsSubscriberCount += (countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_GCLOUD_JAVA_SUBSCRIBER): 0; - cpsSubscriberCount += (countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER) != null) ? countMap.get(ClientType.CPS_EXPERIMENTAL_JAVA_SUBSCRIBER): 0; - kafkaPublisherCount += (countMap.get(ClientType.KAFKA_PUBLISHER) != null) ? countMap.get(ClientType.KAFKA_PUBLISHER) : 0; - kafkaSubscriberCount += (countMap.get(ClientType.KAFKA_SUBSCRIBER) != null) ? countMap.get(ClientType.KAFKA_SUBSCRIBER) : 0; + countMap.forEach((k, v) -> { + if (k.isCpsPublisher()) { + cpsPublisherCount += v; + } else if (k.isKafkaPublisher()) { + kafkaPublisherCount += v; + } else if (k.toString().startsWith("kafka")) { + kafkaSubscriberCount += v; + } else { + cpsSubscriberCount += v; + } + }); }); } @@ -136,12 +137,10 @@ public List>> getValuesList(Map { List valueRow = new ArrayList<>(13); switch (type) { - case CPS_EXPERIMENTAL_JAVA_PUBLISHER: case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: - case CPS_MAPPED_JAVA_PUBLISHER: + case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: - case CPS_VTK_JAVA_PUBLISHER: if (cpsPublisherCount == 0) { return; } @@ -149,8 +148,10 @@ public List>> getValuesList(Map Date: Thu, 12 Oct 2017 18:05:19 -0700 Subject: [PATCH 139/140] Load test framework, producer configs, and a few bugs. (#132) * Add Mapped API tasks, order test, fix correctness test. * Added fixes to sheets, removed comments from tasks. * Refactored producer configs, fixed bugs on consumer side. --- load-test-framework/driver.py | 152 ++++++++++++++++ load-test-framework/pom.xml | 22 ++- load-test-framework/run.py | 1 + .../clients/adapter/PublisherAdapterTask.java | 4 +- .../adapter/SubscriberAdapterTask.java | 4 +- .../pubsub/clients/common/LoadTestRunner.java | 38 ++-- .../google/pubsub/clients/common/Task.java | 73 +++++--- .../clients/gcloud/CPSPublisherTask.java | 89 +++++---- .../clients/gcloud/CPSSubscriberTask.java | 47 ++--- .../clients/kafka/KafkaPublisherTask.java | 47 +++-- .../clients/kafka/KafkaSubscriberTask.java | 28 ++- .../clients/mapped/CPSPublisherTask.java | 97 +++++++--- .../clients/mapped/CPSSubscriberTask.java | 104 +++++++++++ .../java/com/google/pubsub/flic/Driver.java | 79 ++++++-- .../pubsub/flic/controllers/Client.java | 51 ++++-- .../pubsub/flic/controllers/Controller.java | 56 +++++- .../flic/controllers/GCEController.java | 49 +++-- .../flic/controllers/MessageTracker.java | 35 ++-- .../pubsub/flic/output/SheetsService.java | 75 ++++++-- .../src/main/proto/loadtest.proto | 7 + ...ka-mapped-java-publisher_startup_script.sh | 31 ++++ ...a-mapped-java-subscriber_startup_script.sh | 31 ++++ .../pubsub/flic/controllers/ClientTest.java | 5 +- .../pubsub/flic/output/SheetsServiceTest.java | 5 + pubsub-mapped-api/pom.xml | 15 +- .../pubsub/clients/consumer/Config.java | 14 +- .../clients/consumer/KafkaConsumer.java | 32 ++-- .../consumer/PubSubConsumerConfig.java | 8 + .../clients/consumer/ack/Subscriber.java | 17 +- .../pubsub/clients/producer/Config.java | 169 ++++++++++++++---- .../clients/producer/KafkaProducer.java | 151 +++++----------- .../producer/PubSubProducerConfig.java | 51 ++++++ .../com/google/pubsub/common/ChannelUtil.java | 3 +- .../consumer/ConsumerConfigCreator.java | 2 +- ...dapter.java => ProducerConfigCreator.java} | 15 +- .../pubsub/clients/consumer/ConfigTest.java | 1 + .../clients/consumer/KafkaConsumerTest.java | 16 +- .../clients/consumer/ack/SubscriberTest.java | 25 ++- .../pubsub/clients/producer/ConfigTest.java | 31 ++-- .../clients/producer/KafkaProducerTest.java | 82 +++------ .../producer/MultiplyByTenInterceptor.java | 42 +++++ .../producer/ThrowExceptionInterceptor.java | 40 +++++ 42 files changed, 1345 insertions(+), 499 deletions(-) create mode 100755 load-test-framework/driver.py create mode 100644 load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSSubscriberTask.java create mode 100644 load-test-framework/src/main/resources/gce/kafka-mapped-java-publisher_startup_script.sh create mode 100644 load-test-framework/src/main/resources/gce/kafka-mapped-java-subscriber_startup_script.sh create mode 100644 pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java rename pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/{ProducerConfigAdapter.java => ProducerConfigCreator.java} (71%) create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/MultiplyByTenInterceptor.java create mode 100644 pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ThrowExceptionInterceptor.java diff --git a/load-test-framework/driver.py b/load-test-framework/driver.py new file mode 100755 index 00000000..28e39c79 --- /dev/null +++ b/load-test-framework/driver.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import os +import sys +import getopt +import subprocess + +def main(project, test, vms_count, zone, sheet_id, + broker, zookeeper, partitions, replication, mapped, cps): + + subprocess.call(['mvn', 'clean', 'install'], cwd='../pubsub-mapped-api') + subprocess.call(['mvn', 'clean', 'package']) + + if not os.path.isfile('./src/main/resources/gce/driver.jar'): + subprocess.call(['cp', 'target/driver.jar', 'target/classes/gce/']) + + arg_list = ['java', '-Xmx32G', '-jar', 'target/driver.jar', '--project', project] + + # CPS + if cps: + arg_list.append('--cps_gcloud_java_publisher_count=' + str(vms_count)) + arg_list.append('--cps_gcloud_java_subscriber_count=' + str(vms_count)) + + # Kafka + if len(broker) != 0: + arg_list.append('--broker=' + broker) + arg_list.append('--kafka_publisher_count=' + str(vms_count)) + arg_list.append('--kafka_subscriber_count=' + str(vms_count)) + arg_list.append('--partitions=' + str(partitions)) + arg_list.append('--replication_factor=' + str(replication)) + + if len(zookeeper) != 0: + arg_list.append('--zookeeper_ip_address=' + zookeeper) + + # Mapped + if mapped: + arg_list.append('--kafka_mapped_java_publisher_count=' + str(vms_count)) + arg_list.append('--kafka_mapped_java_subscriber_count=' + str(vms_count)) + + if test == 'latency': + arg_list.extend([ + '--message_size=1', '--publish_batch_size=1', '--request_rate=1', + '--max_outstanding_requests=10', '--loadtest_duration=10m', + '--burn_in_duration=2m', '--publish_batch_duration=1ms' + ]) + elif test == 'throughput': + arg_list.extend([ + '--message_size=10000', '--publish_batch_size=10', + '--request_rate=1000000000', '--max_outstanding_requests=1600', + '--loadtest_duration=10m', '--burn_in_duration=2m', + '--publish_batch_duration=50ms', '--num_cores_test' + ]) + elif test == 'test_throughput': + arg_list.extend([ + '--message_size=10000', '--publish_batch_size=10', + '--request_rate=1000000000', '--max_outstanding_requests=200', + '--loadtest_duration=2m', '--burn_in_duration=1m', + '--publish_batch_duration=50ms', '--cores=2' + ]) + elif test == 'service': + arg_list.extend([ + '--message_size=10000', '--publish_batch_size=10', + '--request_rate=1000000000', '--max_outstanding_requests=1600', + '--loadtest_duration=10m', '--burn_in_duration=2m', + '--publish_batch_duration=50ms', '--cores=16', + ]) + elif test == 'ordering': + arg_list.extend([ + '--order_test', '--message_size=1', + '--number_of_messages=100000', '--publish_batch_size=100', + '--request_rate=1000', '--max_outstanding_requests=100', + '--burn_in_duration=1m', '--publish_batch_duration=1m' + ]) + elif test == 'correctness': + arg_list.extend([ + '--message_size=1', '--publish_batch_duration=30s', + '--number_of_messages=1000000', '--publish_batch_size=1', + '--request_rate=10000', '--max_outstanding_requests=1000', + '--burn_in_duration=1m' + ]) + + if len(sheet_id) != 0: + arg_list.append('--spreadsheet_id=' + sheet_id) + + arg_list.append('--zone=' + zone) + + print(' '.join(arg_list)) + subprocess.call(arg_list) + + +if __name__ == '__main__': + + broker = '' + sheet_id = '' + zookeeper = '' + partitions = 0 + replication = 0 + vms_count = 1 + project = None + test = 'latency' + zone = 'us-central1-a' + + cps = False + mapped = False + + opts, _ = getopt.getopt( + sys.argv[1:], '', + ['vms_count=', 'test=', 'project=', 'zone=', + 'sheet_id=', 'broker=', 'zookeeper=', 'partitions=', 'replication=', + 'mapped', 'cps']) + + for opt, arg in opts: + if opt == '--test': + test = arg + if opt == '--project': + project = arg + if opt == '--vms_count': + vms_count = int(arg) + if opt == '--partitions': + partitions = int(arg) + if opt == '--replication': + replication = int(arg) + if opt == '--zone': + zone = arg + if opt == '--sheet_id': + sheet_id = arg + if opt == '--cps': + cps = True + if opt == '--broker': + broker = arg + if opt == '--zookeeper': + zookeeper = arg + if opt == '--mapped': + mapped = True + + if not cps and not mapped and len(broker) == 0: + cps = True + if not project: + sys.exit('You must provide the name of your project with --project.') + if vms_count < 1: + sys.exit('If provided, --vms_count must be greater than 0.') + if len(broker) != 0 and partitions < 1: + sys.exit('If provided, --partitions must be greater than 0.') + if len(broker) != 0 and replication < 1: + sys.exit('If provided, --replication must be greater than 0.') + if test not in ['latency', 'service', + 'throughput', 'test_throughput', + 'ordering', 'correctness']: + sys.exit('Invalid --test parameter given.') + + main(project, test, vms_count, zone, sheet_id, + broker, zookeeper, partitions, replication, mapped, cps) \ No newline at end of file diff --git a/load-test-framework/pom.xml b/load-test-framework/pom.xml index 57503691..482f101a 100644 --- a/load-test-framework/pom.xml +++ b/load-test-framework/pom.xml @@ -16,18 +16,13 @@ org.apache.kafka kafka_2.11 - 0.10.0.0 + 0.10.1.0 com.beust jcommander 1.48 - - com.google.cloud - google-cloud-pubsub - 0.24.0-beta - commons-io commons-io @@ -119,6 +114,21 @@ zkclient 0.7 + + org.apache.kafka + kafka-clients + 0.10.1.0 + + + com.google.cloud + google-cloud-pubsub + 0.20.3-beta + + + com.google.pubsub + pubsub-mapped-api + 1.0-SNAPSHOT + diff --git a/load-test-framework/run.py b/load-test-framework/run.py index 43030156..3ebdebb9 100644 --- a/load-test-framework/run.py +++ b/load-test-framework/run.py @@ -104,6 +104,7 @@ def main(project, test, client_types, vms_count, broker): '--loadtest_duration=10m', '--burn_in_duration=2m', '--publish_batch_duration=50ms', '--cores=16' ]) + print(' '.join(arg_list)) subprocess.call(arg_list) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/PublisherAdapterTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/PublisherAdapterTask.java index 410ee797..17d7486d 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/PublisherAdapterTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/PublisherAdapterTask.java @@ -25,13 +25,13 @@ */ class PublisherAdapterTask extends AdapterTask { - private PublisherAdapterTask(LoadtestProto.StartRequest request, AdapterTask.Options options) { + private PublisherAdapterTask(LoadtestProto.StartRequest request, Options options) { super(request, MetricsHandler.MetricName.PUBLISH_ACK_LATENCY, options); } public static void main(String[] args) throws Exception { LoadTestRunner.Options loadtestOptions = new LoadTestRunner.Options(); - AdapterTask.Options adapterOptions = new AdapterTask.Options(); + Options adapterOptions = new Options(); new JCommander(loadtestOptions, args); new JCommander(adapterOptions, args); LoadTestRunner.run( diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/SubscriberAdapterTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/SubscriberAdapterTask.java index eea27353..4341f11b 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/SubscriberAdapterTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/adapter/SubscriberAdapterTask.java @@ -23,13 +23,13 @@ /** A Task that proxies subscriber commands to a LoadtestWorker process on localhost. */ class SubscriberAdapterTask extends AdapterTask { - private SubscriberAdapterTask(LoadtestProto.StartRequest request, AdapterTask.Options options) { + private SubscriberAdapterTask(LoadtestProto.StartRequest request, Options options) { super(request, MetricsHandler.MetricName.END_TO_END_LATENCY, options); } public static void main(String[] args) throws Exception { LoadTestRunner.Options loadtestOptions = new LoadTestRunner.Options(); - AdapterTask.Options adapterOptions = new AdapterTask.Options(); + Options adapterOptions = new Options(); new JCommander(loadtestOptions, args); new JCommander(adapterOptions, args); LoadTestRunner.run( diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java b/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java index 9749d23d..74dffac0 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java @@ -18,26 +18,35 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + import com.google.common.base.Stopwatch; import com.google.common.util.concurrent.SettableFuture; + import com.google.protobuf.Duration; import com.google.protobuf.util.Timestamps; -import com.google.pubsub.flic.common.LoadtestGrpc; + +import com.google.pubsub.flic.common.LoadtestGrpc.LoadtestImplBase; +import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import com.google.pubsub.flic.common.LoadtestProto.CheckRequest; import com.google.pubsub.flic.common.LoadtestProto.CheckResponse; -import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import com.google.pubsub.flic.common.LoadtestProto.StartResponse; +import com.google.pubsub.flic.common.LoadtestProto.StartRequest.StopConditionsCase; + import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.stub.StreamObserver; + import java.nio.charset.Charset; + +import java.util.function.Function; + import java.util.Arrays; -import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.Executors; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +54,13 @@ * Starts a server to get the start request, then starts the load task. */ public class LoadTestRunner { - private static final Logger log = LoggerFactory.getLogger(LoadTestRunner.class); + private static Task task; + private static Server server; private static final int MAX_IDLE_MILLIS = 60 * 1000; // 1 minute private static final Stopwatch stopwatch = Stopwatch.createUnstarted(); - private static Server server; - private static Task task; - private static final AtomicBoolean finished = new AtomicBoolean(true); private static SettableFuture finishedFuture = SettableFuture.create(); + private static final Logger log = LoggerFactory.getLogger(LoadTestRunner.class); + private static final AtomicBoolean finished = new AtomicBoolean(true); /** * Command line options for the {@link LoadTestRunner}. @@ -85,16 +94,16 @@ private static void runTest(StartRequest request) { log.error("Interrupted sleeping, starting test now."); } } - stopwatch.start(); while (shouldContinue(request)) { executor.execute(task); } + log.info("Load test complete."); stopwatch.stop(); finished.set(true); task.shutdown(); executor.shutdownNow(); - log.info("Load test complete."); + log.info("Shutting down server."); } public static void run(Options options, Function function) @@ -114,7 +123,7 @@ public static void run( server = serverBuilder .addService( - new LoadtestGrpc.LoadtestImplBase() { + new LoadtestImplBase() { @Override public void start( StartRequest request, StreamObserver responseObserver) { @@ -140,9 +149,9 @@ public void check( .setRunningDuration( Duration.newBuilder() .setSeconds(stopwatch.elapsed(TimeUnit.SECONDS))) - .setIsFinished(finishedValue) .addAllReceivedMessages( task.flushMessageIdentifiers(request.getDuplicatesList())) + .setIsFinished(finishedValue) .build()); responseObserver.onCompleted(); if (finishedValue) { @@ -181,7 +190,8 @@ private static boolean shouldContinue(StartRequest request) { + request.getTestDuration().getSeconds()) * 1000; case NUMBER_OF_MESSAGES: - return task.getNumberOfMessages() < request.getNumberOfMessages(); + return task.getNumberOfMessages() < request.getNumberOfMessages() + && task.getActualCounter() < request.getNumberOfMessages(); default: return false; } diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java b/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java index cdbcd154..1682c239 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java @@ -16,37 +16,48 @@ package com.google.pubsub.clients.common; -import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.util.concurrent.FutureCallback; +import com.google.common.base.Preconditions; + +import com.google.protobuf.util.Timestamps; + import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.RateLimiter; -import com.google.protobuf.util.Timestamps; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.ListenableFuture; + +import com.google.pubsub.flic.controllers.Client; import com.google.pubsub.flic.common.LoadtestProto.MessageIdentifier; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; + import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.Comparator; import java.util.Map; -import java.util.concurrent.Semaphore; +import java.util.List; +import java.util.Map.Entry; +import java.util.LinkedHashMap; +import java.util.stream.Collectors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicInteger; /** * Each task is responsible for implementing its action and for creating {@link LoadTestRunner}. */ public abstract class Task implements Runnable { private final MetricsHandler metricsHandler; - private AtomicInteger numMessages = new AtomicInteger(0); - private final Map identifiers = new HashMap<>(); - private final Map identifiersToRecord = new HashMap<>(); - private final AtomicLong lastUpdateMillis = new AtomicLong(System.currentTimeMillis()); + private final Map identifiers = new LinkedHashMap<>(); + private final Map identifiersToRecord = new LinkedHashMap<>(); private final long burnInTimeMillis; private final RateLimiter rateLimiter; private final Semaphore outstandingRequestLimiter; + private final AtomicInteger numMessages = new AtomicInteger(0); + private final AtomicLong lastUpdateMillis = new AtomicLong(System.currentTimeMillis()); + + protected final AtomicInteger actualCounter = new AtomicInteger(0); + protected Task(StartRequest request, String type, MetricsHandler.MetricName metricName) { this.metricsHandler = new MetricsHandler(request.getProject(), type, metricName); this.burnInTimeMillis = @@ -65,11 +76,14 @@ public static class RunResult { public List latencies = new ArrayList<>(); public int batchSize = 0; - public void addMessageLatency(int clientId, int sequenceNumber, long latency) { + public void addMessageLatency( + int clientId, int sequenceNumber, long publishTime, long receiveTime, long latency) { identifiers.add(MessageIdentifier.newBuilder() - .setPublisherClientId(clientId) - .setSequenceNumber(sequenceNumber) - .build()); + .setPublisherClientId(clientId) + .setSequenceNumber(sequenceNumber) + .setPublishTime(publishTime) + .setReceiveTime(receiveTime) + .build()); latencies.add(latency); } @@ -139,7 +153,11 @@ List getBucketValues() { return metricsHandler.flushBucketValues(); } - int getNumberOfMessages() { + protected int getActualCounter() { + return actualCounter.get(); + } + + protected int getNumberOfMessages() { return numMessages.get(); } @@ -148,23 +166,26 @@ private void recordLatency(long millis) { } protected void recordBatchLatency(long millis, int batchSize) { - lastUpdateMillis.set(System.currentTimeMillis()); if (System.currentTimeMillis() < burnInTimeMillis) { return; } - metricsHandler.recordBatchLatency(millis, batchSize); numMessages.getAndAdd(batchSize); + lastUpdateMillis.set(System.currentTimeMillis()); + metricsHandler.recordBatchLatency(millis, batchSize); } long getLastUpdateMillis() { return lastUpdateMillis.get(); } - protected synchronized void recordMessageLatency(int clientId, int sequenceNumber, long latency) { + protected synchronized void recordMessageLatency( + int clientId, int sequenceNumber, long publishTime, long receiveTime, long latency) { identifiers.put( MessageIdentifier.newBuilder() .setPublisherClientId(clientId) .setSequenceNumber(sequenceNumber) + .setPublishTime(publishTime) + .setReceiveTime(receiveTime) .build(), latency); lastUpdateMillis.set(System.currentTimeMillis()); @@ -180,6 +201,7 @@ protected synchronized void recordAllMessageLatencies( for (int i = 0; i < identifiers.size(); i++) { this.identifiers.put(identifiers.get(i), latencies.get(i)); } + numMessages.getAndAdd(identifiers.size()); lastUpdateMillis.set(System.currentTimeMillis()); } @@ -190,6 +212,17 @@ synchronized List flushMessageIdentifiers(List entry.getKey().getReceiveTime())) + .collect(Collectors.toMap( + Entry::getKey, + Entry::getValue, + (x,y)-> {throw new AssertionError();}, + LinkedHashMap::new)); + } identifiersToRecord.putAll(identifiers); identifiers.clear(); return new ArrayList<>(identifiersToRecord.keySet()); diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java index 2dcb5573..c7f17269 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSPublisherTask.java @@ -16,44 +16,54 @@ package com.google.pubsub.clients.gcloud; import com.beust.jcommander.JCommander; -import com.google.api.core.ApiFutureCallback; + import com.google.api.core.ApiFutures; +import com.google.api.core.ApiFutureCallback; import com.google.api.gax.batching.BatchingSettings; -import com.google.cloud.pubsub.v1.Publisher; +import com.google.api.gax.batching.FlowControlSettings; + import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import com.google.common.util.concurrent.ListenableFuture; + import com.google.protobuf.ByteString; import com.google.protobuf.util.Durations; + +import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.common.LoadTestRunner; import com.google.pubsub.clients.common.MetricsHandler; -import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.common.Task.RunResult; -import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.v1.PubsubMessage; + +import com.google.cloud.pubsub.v1.Publisher; + import com.google.pubsub.v1.TopicName; +import com.google.pubsub.v1.PubsubMessage; + +import com.google.pubsub.flic.common.LoadtestProto.StartRequest; + import java.util.Random; -import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import org.threeten.bp.Duration; /** * Runs a task that publishes messages to a Cloud Pub/Sub topic. */ class CPSPublisherTask extends Task { - private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); - private final Publisher publisher; - private final ByteString payload; - private final int batchSize; + private final Integer id; + private final int batchSize; + private final ByteString payload; + private final Publisher publisher; + private final AtomicInteger sending = new AtomicInteger(0); private final AtomicInteger sequenceNumber = new AtomicInteger(0); - private final Semaphore outstandingBytes = new Semaphore(1000000000); // 1 GB - private final int messageSize; + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); private CPSPublisherTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); + this.id = (new Random()).nextInt(); + this.batchSize = request.getPublishBatchSize(); + this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); try { this.publisher = Publisher.defaultBuilder(TopicName.create(request.getProject(), request.getTopic())) @@ -63,15 +73,16 @@ private CPSPublisherTask(StartRequest request) { .setRequestByteThreshold(9500000L) .setDelayThreshold( Duration.ofMillis(Durations.toMillis(request.getPublishBatchDuration()))) + + .setFlowControlSettings(FlowControlSettings.newBuilder() + .setMaxOutstandingRequestBytes(1000 * 1000 * 1000L) + .build()) + .build()) .build(); } catch (Exception e) { throw new RuntimeException(e); } - this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); - this.batchSize = request.getPublishBatchSize(); - this.messageSize = request.getMessageSize(); - this.id = (new Random()).nextInt(); } public static void main(String[] args) throws Exception { @@ -82,43 +93,51 @@ public static void main(String[] args) throws Exception { @Override public ListenableFuture doRun() { + AtomicInteger numPending = new AtomicInteger(batchSize); - final SettableFuture done = SettableFuture.create(); String sendTime = String.valueOf(System.currentTimeMillis()); - if (!outstandingBytes.tryAcquire(batchSize * messageSize)) { - return Futures.immediateFailedFuture(new Exception("Flow control limits reached.")); + final SettableFuture done = SettableFuture.create(); + + if (shuttingDown.get()) { + return Futures.immediateFailedFuture( + new IllegalStateException("the task is shutting down.")); } + + sending.incrementAndGet(); for (int i = 0; i < batchSize; i++) { - ApiFutures.addCallback(publisher - .publish( + actualCounter.incrementAndGet(); + ApiFutures.addCallback( + publisher.publish( PubsubMessage.newBuilder() - .setData(payload) - .putAttributes("sendTime", sendTime) - .putAttributes("clientId", id.toString()) - .putAttributes( - "sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) - .build()), new ApiFutureCallback() { + .setData(payload) + .putAttributes("sendTime", sendTime) + .putAttributes("clientId", id.toString()) + .putAttributes("sequenceNumber", Integer.toString(sequenceNumber.getAndIncrement())) + .build()), + new ApiFutureCallback() { @Override public void onSuccess(String messageId) { - outstandingBytes.release(messageSize); if (numPending.decrementAndGet() == 0) { done.set(RunResult.fromBatchSize(batchSize)); } } - @Override public void onFailure(Throwable t) { - outstandingBytes.release(messageSize); done.setException(t); } }); } + sending.decrementAndGet(); return done; } @Override - protected void shutdown() { + public void shutdown() { try { + if (shuttingDown.getAndSet(true)) { + return; + } + while (sending.get() > 0) {} publisher.shutdown(); } catch (Exception e) { throw new RuntimeException(e); diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java index ab8be884..047206b9 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/gcloud/CPSSubscriberTask.java @@ -16,48 +16,58 @@ package com.google.pubsub.clients.gcloud; import com.beust.jcommander.JCommander; -import com.google.cloud.pubsub.v1.AckReplyConsumer; -import com.google.cloud.pubsub.v1.MessageReceiver; -import com.google.cloud.pubsub.v1.Subscriber; + import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; + +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.SubscriptionName; + +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.AckReplyConsumer; + +import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.common.LoadTestRunner; import com.google.pubsub.clients.common.MetricsHandler; -import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.common.Task.RunResult; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.SubscriptionName; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Runs a task that consumes messages from a Cloud Pub/Sub subscription. */ class CPSSubscriberTask extends Task implements MessageReceiver { - private static final Logger log = LoggerFactory.getLogger(CPSSubscriberTask.class); - private final SubscriptionName subscription; + private Subscriber subscriber; private boolean shuttingDown = false; + private final SubscriptionName subscription; + private static final Logger log = LoggerFactory.getLogger(CPSSubscriberTask.class); private CPSSubscriberTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.END_TO_END_LATENCY); this.subscription = SubscriptionName.create(request.getProject(), request.getPubsubOptions().getSubscription()); try { - this.subscriber = - Subscriber.defaultBuilder(this.subscription, this) - .setParallelPullCount(Runtime.getRuntime().availableProcessors() * 5) - .build(); + this.subscriber = Subscriber.defaultBuilder(this.subscription, this).build(); } catch (Exception e) { throw new RuntimeException(e); } } + private long getMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000 + ts.getNanos() / 1000000; + } + @Override public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) { + long receiveTime = System.currentTimeMillis(); + long publishTime = getMillis(message.getPublishTime()); recordMessageLatency( Integer.parseInt(message.getAttributesMap().get("clientId")), Integer.parseInt(message.getAttributesMap().get("sequenceNumber")), - System.currentTimeMillis() - Long.parseLong(message.getAttributesMap().get("sendTime"))); + publishTime, + receiveTime, + receiveTime - Long.parseLong(message.getAttributesMap().get("sendTime"))); consumer.ack(); } @@ -74,8 +84,7 @@ public ListenableFuture doRun() { return Futures.immediateFuture(RunResult.empty()); } if (shuttingDown) { - return Futures.immediateFailedFuture( - new IllegalStateException("the task is shutting down")); + return Futures.immediateFailedFuture(new IllegalStateException("The task is shutting down.")); } try { subscriber.startAsync().awaitRunning(); @@ -93,13 +102,11 @@ public void shutdown() { Subscriber subscriber; synchronized (this) { if (shuttingDown) { - throw new IllegalStateException("the task is already shutting down"); + throw new IllegalStateException("The task is already shutting down."); } shuttingDown = true; subscriber = this.subscriber; } - // We must stop out of the lock. Stopping waits for all messages to be processed, - // and processing the messages needs to lock. subscriber.stopAsync().awaitTerminated(); } -} +} \ No newline at end of file diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaPublisherTask.java index 823792f7..0b54a441 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaPublisherTask.java @@ -27,6 +27,8 @@ import com.google.pubsub.clients.common.Task.RunResult; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import java.util.Properties; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -40,27 +42,34 @@ class KafkaPublisherTask extends Task { private static final Logger log = LoggerFactory.getLogger(KafkaPublisherTask.class); + + private final int batchSize; + private final String topic; private final String payload; - private final int batchSize; + private final String clientId; + private final AtomicInteger sequenceNumber = new AtomicInteger(0); + + private final int partitions; private final KafkaProducer publisher; private KafkaPublisherTask(StartRequest request) { super(request, "kafka", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); - this.payload = LoadTestRunner.createMessage(request.getMessageSize()); this.batchSize = request.getPublishBatchSize(); + this.clientId = Integer.toString((new Random()).nextInt()); + this.partitions = request.getKafkaOptions().getPartitions(); + this.payload = LoadTestRunner.createMessage(request.getMessageSize()); Properties props = new Properties(); props.putAll(new ImmutableMap.Builder<>() + .put("acks", "all") .put("max.block.ms", "30000") .put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") .put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - .put("acks", "all") .put("bootstrap.servers", request.getKafkaOptions().getBroker()) .put("buffer.memory", Integer.toString(1000 * 1000 * 1000)) // 1 GB - // 10M, high enough to allow for duration to control batching - .put("batch.size", Integer.toString(10 * 1000 * 1000)) .put("linger.ms", Long.toString(Durations.toMillis(request.getPublishBatchDuration()))) + .put("batch.size", Integer.toString(request.getMessageSize() * request.getPublishBatchSize())) .build() ); this.publisher = new KafkaProducer<>(props); @@ -74,22 +83,32 @@ public static void main(String[] args) throws Exception { @Override public ListenableFuture doRun() { - SettableFuture result = SettableFuture.create(); + + AtomicInteger messagesSent = new AtomicInteger(batchSize); AtomicInteger messagesToSend = new AtomicInteger(batchSize); - AtomicInteger messagesSentSuccess = new AtomicInteger(batchSize); + String sendTime = String.valueOf(System.currentTimeMillis()); + final SettableFuture done = SettableFuture.create(); + for (int i = 0; i < batchSize; i++) { + actualCounter.incrementAndGet(); publisher.send( - new ProducerRecord<>(topic, null, System.currentTimeMillis(), null, payload), - (metadata, exception) -> { - if (exception != null) { - messagesSentSuccess.decrementAndGet(); - log.error(exception.getMessage(), exception); + new ProducerRecord<>(topic, sequenceNumber.get() % partitions, clientId + "#" + + Integer.toString(sequenceNumber.getAndIncrement()) + "#" + sendTime, payload), + (recordMetadata, e) -> { + if (e != null) { + messagesSent.getAndDecrement(); + log.error(e.getMessage(), e); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + done.set(RunResult.fromBatchSize(messagesSent.get())); } }); } - return result; + return done; + } + + @Override + public void shutdown() { + publisher.close(); } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaSubscriberTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaSubscriberTask.java index 619ae6a1..a3d02ca7 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaSubscriberTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/kafka/KafkaSubscriberTask.java @@ -23,10 +23,10 @@ import com.google.pubsub.clients.common.LoadTestRunner; import com.google.pubsub.clients.common.MetricsHandler; import com.google.pubsub.clients.common.Task; -import com.google.pubsub.clients.common.Task.RunResult; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import java.util.Collections; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -35,17 +35,20 @@ * interface. */ class KafkaSubscriberTask extends Task { + private final long pollLength; private final KafkaConsumer subscriber; private KafkaSubscriberTask(StartRequest request) { super(request, "kafka", MetricsHandler.MetricName.END_TO_END_LATENCY); + this.pollLength = Durations.toMillis(request.getKafkaOptions().getPollDuration()); + Properties props = new Properties(); props.putAll(ImmutableMap.of( + "group.id", "SUBSCRIBER_ID", "key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", - "group.id", "SUBSCRIBER_ID", "enable.auto.commit", "true", "session.timeout.ms", "30000" )); @@ -62,10 +65,23 @@ public static void main(String[] args) throws Exception { @Override public ListenableFuture doRun() { - RunResult result = new RunResult(); ConsumerRecords records = subscriber.poll(pollLength); - long now = System.currentTimeMillis(); - records.forEach(record -> result.latencies.add(now - record.timestamp())); - return Futures.immediateFuture(result); + records.forEach( + record -> { + String[] tokens = record.key().split("#"); + long receiveTime = System.currentTimeMillis(); + recordMessageLatency( + Integer.parseInt(tokens[0]), + Integer.parseInt(tokens[1]), + record.timestamp(), + receiveTime, + receiveTime - Long.parseLong(tokens[2])); + }); + return Futures.immediateFuture(RunResult.empty()); + } + + @Override + public void shutdown() { + subscriber.close(); } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java index 2890aae0..9dfc00e8 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSPublisherTask.java @@ -1,35 +1,64 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + package com.google.pubsub.clients.mapped; import com.beust.jcommander.JCommander; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.util.Durations; + import com.google.pubsub.clients.common.Task; import com.google.pubsub.clients.common.LoadTestRunner; +import com.google.pubsub.clients.common.MetricsHandler.MetricName; + import com.google.pubsub.clients.producer.KafkaProducer; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; -import com.google.pubsub.clients.common.MetricsHandler.MetricName; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.kafka.clients.producer.ProducerRecord; + +import java.util.Random; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.kafka.clients.producer.ProducerRecord; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer - * interface + * Runs a task that publishes messages utilizing Pub/Sub's implementation of the Kafka Producer Interface. */ -public class CPSPublisherTask extends Task { +class CPSPublisherTask extends Task { private static final Logger log = LoggerFactory.getLogger(CPSPublisherTask.class); + private final int batchSize; + private final int messageSize; + private final long batchDelay; + private final String topic; private final String payload; - private final int batchSize; + private final String clientId; + private final AtomicInteger sending = new AtomicInteger(0); + private final AtomicInteger sequenceNumber = new AtomicInteger(0); + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); + private final KafkaProducer publisher; @SuppressWarnings("unchecked") @@ -37,11 +66,19 @@ private CPSPublisherTask(StartRequest request) { super(request, "mapped", MetricName.PUBLISH_ACK_LATENCY); this.topic = request.getTopic(); + this.messageSize = request.getMessageSize(); this.batchSize = request.getPublishBatchSize(); - this.payload = LoadTestRunner.createMessage(request.getMessageSize()); + this.payload = LoadTestRunner.createMessage(messageSize); + this.clientId = Integer.toString((new Random()).nextInt()); + this.batchDelay = Durations.toMillis(request.getPublishBatchDuration()); Properties props = new Properties(); - props.put("project", "dataproc-kafka-test"); + props.put("project", request.getProject()); + props.put("linger.ms", Long.toString(batchDelay)); + props.put("elements.count", Integer.toString(batchSize)); + props.put("batch.size", Integer.toString(9500000)); + props.put("client.id", clientId); + props.put("buffer.memory", Integer.toString(1000 * 1000 * 1000)); props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); @@ -50,37 +87,55 @@ private CPSPublisherTask(StartRequest request) { public static void main(String[] args) throws Exception { LoadTestRunner.Options options = new LoadTestRunner.Options(); - new JCommander(options, args); - LoadTestRunner.run(options, CPSPublisherTask::new); } @Override public ListenableFuture doRun() { - SettableFuture result = SettableFuture.create(); + AtomicInteger messagesSent = new AtomicInteger(batchSize); AtomicInteger messagesToSend = new AtomicInteger(batchSize); - AtomicInteger messagesSentSuccess = new AtomicInteger(batchSize); + String sendTime = String.valueOf(System.currentTimeMillis()); + final SettableFuture done = SettableFuture.create(); + + if (shuttingDown.get()) { + return Futures.immediateFailedFuture( + new IllegalStateException("The task is shutting down.")); + } + sending.incrementAndGet(); for (int i = 0; i < batchSize; i++) { + actualCounter.incrementAndGet(); publisher.send( - new ProducerRecord<>(topic, 0, "", payload), - (metadata, exception) -> { - if (exception != null) { - messagesSentSuccess.decrementAndGet(); - log.error(exception.getMessage(), exception); + new ProducerRecord<>(topic, 0, clientId + "#" + + Integer.toString(sequenceNumber.getAndIncrement()) + "#" + sendTime, payload), + (recordMetadata, e) -> { + if (e != null) { + messagesSent.getAndDecrement(); + log.error(e.getMessage(), e); } if (messagesToSend.decrementAndGet() == 0) { - result.set(RunResult.fromBatchSize(messagesSentSuccess.get())); + done.set(RunResult.fromBatchSize(messagesSent.get())); } }); } - return result; + sending.decrementAndGet(); + return done; } @Override public void shutdown() { - publisher.close(); + try { + if (shuttingDown.getAndSet(true)) { + return; + } + while (sending.get() > 0) { + //Don't shutdown until all the ongoing processes finish + } + publisher.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSSubscriberTask.java b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSSubscriberTask.java new file mode 100644 index 00000000..c19c4aa7 --- /dev/null +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/mapped/CPSSubscriberTask.java @@ -0,0 +1,104 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.mapped; + +import com.beust.jcommander.JCommander; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import com.google.pubsub.clients.common.Task; +import com.google.pubsub.clients.common.LoadTestRunner; +import com.google.pubsub.clients.consumer.KafkaConsumer; +import com.google.pubsub.clients.common.MetricsHandler.MetricName; + +import com.google.pubsub.flic.common.LoadtestProto.StartRequest; + +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.clients.consumer.ConsumerRecords; + +import java.util.Properties; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Runs a task that consumes messages utilizing Pub/Sub's implementation of the Kafka Consumer Interface. + */ +class CPSSubscriberTask extends Task { + + private static final long POLL_TIMEOUT = 300000L; + private final KafkaConsumer subscriber; + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); + + private CPSSubscriberTask(StartRequest request) { + super(request, "mapped", MetricName.END_TO_END_LATENCY); + + Properties props = new Properties(); + + //TODO: DON'T CHANGE, ELSE YOU WILL BREAK THE TEST. Subscription name depends on it. + props.put("group.id", "SUBSCRIBER_ID"); + props.put("enable.auto.commit", "false"); + props.put("project", request.getProject()); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + + this.subscriber = new KafkaConsumer<>(props); + this.subscriber.subscribe(Collections.singletonList(request.getTopic())); + } + + public static void main(String[] args) throws Exception { + LoadTestRunner.Options options = new LoadTestRunner.Options(); + new JCommander(options, args); + LoadTestRunner.run(options, CPSSubscriberTask::new); + } + + @Override + public ListenableFuture doRun() { + if (shuttingDown.get()) { + return Futures.immediateFailedFuture( + new IllegalStateException("The task is shutting down.")); + } + ConsumerRecords records = null; + try { + records = subscriber.poll(POLL_TIMEOUT); + } catch (KafkaException e) { + return Futures.immediateFailedFuture( + new IllegalStateException("The task is shut down.")); + } + records.forEach( + record -> { + String[] tokens = record.key().split("#"); + long receiveTime = System.currentTimeMillis(); + recordMessageLatency( + Integer.parseInt(tokens[0]), + Integer.parseInt(tokens[1]), + record.timestamp(), + receiveTime, + receiveTime - Long.parseLong(tokens[2])); + }); + subscriber.commitAsync(); + return Futures.immediateFuture(RunResult.empty()); + } + + @Override + public void shutdown() { + if (shuttingDown.getAndSet(true)) { + return; + } + subscriber.close(); + } +} diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index fbe16b66..35732eaa 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -36,6 +36,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.util.concurrent.AtomicDouble; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; @@ -142,6 +143,18 @@ public class Driver { ) private int kafkaSubscriberCount = 0; + @Parameter( + names = {"--kafka_mapped_java_publisher_count"}, + description = "Number of mapped publishers to start." + ) + private int kafkaMappedJavaPublisherCount = 0; + + @Parameter( + names = {"--kafka_mapped_java_subscriber_count"}, + description = "Number of mapped subscribers to start." + ) + private int kafkaMappedJavaSubscriberCount = 0; + @Parameter( names = {"--message_size", "-m"}, description = "Message size in bytes (only when publishing messages).", @@ -171,9 +184,9 @@ public class Driver { private int publishBatchSize = 10; @Parameter( - names = {"--publish_batch_duration"}, - description = "The maximum duration to wait for more messages to batch together.", - converter = DurationConverter.class + names = {"--publish_batch_duration"}, + description = "The maximum duration to wait for more messages to batch together.", + converter = DurationConverter.class ) private Duration publishBatchDuration = Durations.fromMillis(0); @@ -233,7 +246,7 @@ public class Driver { names = {"--number_of_messages"}, description = "The total number of messages to publish in the test. Enabling this will " - + "override --loadtest_length_seconds. Enabling this flag will also enable the check " + + "override --loadtest_duration. Enabling this flag will also enable the check " + "for message loss. If set less than 1, this flag is ignored." ) private int numberOfMessages = 0; @@ -289,6 +302,14 @@ public class Driver { ) private boolean numCoresTest = false; + @Parameter( + names = {"--order_test"}, + description = + "This is a test which measures the ordering degree of the received msgs compared" + + "to how they were sent." + ) + private boolean orderTest = false; + @Parameter( names = {"--spreadsheet_id"}, description = "The id of the spreadsheet to which results are output." @@ -345,6 +366,7 @@ public class Driver { private int partitions = 100; private Controller controller; + private static ScheduledExecutorService scheduledExecutor; public static void main(String[] args) { Driver driver = new Driver(); @@ -353,13 +375,14 @@ public static void main(String[] args) { jCommander.usage(); return; } + scheduledExecutor = Executors.newScheduledThreadPool(500); driver.run( (project, clientParamsMap) -> GCEController.newGCEController( project, ImmutableMap.of(driver.zone, clientParamsMap), driver.cores, - Executors.newScheduledThreadPool(500))); + scheduledExecutor)); } public void run(BiFunction, Controller> controllerFunction) { @@ -385,6 +408,11 @@ public void run(BiFunction, Controller> contr new ClientParams(ClientType.CPS_GCLOUD_GO_PUBLISHER, null), cpsGcloudGoPublisherCount); } + if (kafkaMappedJavaPublisherCount > 0) { + clientParamsMap.put( + new ClientParams(ClientType.KAFKA_MAPPED_JAVA_PUBLISHER, null), + kafkaMappedJavaPublisherCount); + } if (kafkaPublisherCount > 0) { clientParamsMap.put( new ClientParams(ClientType.KAFKA_PUBLISHER, null), kafkaPublisherCount); @@ -410,6 +438,7 @@ public void run(BiFunction, Controller> contr clientParamsMap.size() == 1, "If max_publish_latency is specified, there must be one type of publisher."); } + // Each type of client will have its own topic, so each topic will get // cpsSubscriberCount subscribers cumulatively among each of the subscriptions. for (int i = 0; i < cpsSubscriptionFanout; ++i) { @@ -441,6 +470,14 @@ public void run(BiFunction, Controller> contr new ClientParams(ClientType.CPS_GCLOUD_RUBY_SUBSCRIBER, "gcloud-ruby-subscription" + i), cpsGcloudRubySubscriberCount / cpsSubscriptionFanout); } + //TODO: fix the subscription name - it won't create more than one - + if (kafkaMappedJavaSubscriberCount > 0) { + Preconditions.checkArgument(kafkaMappedJavaPublisherCount > 0, + "--kafka_mapped_java_publisher must be > 0."); + clientParamsMap.put( + new ClientParams(ClientType.KAFKA_MAPPED_JAVA_SUBSCRIBER, "cloud-pubsub-loadtest-mapped_SUBSCRIBER_ID"), + kafkaMappedJavaSubscriberCount / cpsSubscriptionFanout); + } } // Set static variables. Controller.resourceDirectory = resourceDirectory; @@ -458,6 +495,8 @@ public void run(BiFunction, Controller> contr Client.publishBatchDuration = publishBatchDuration; Client.partitions = partitions; Client.replicationFactor = replicationFactor; + Client.cores = cores; + Client.orderTest = orderTest; // Start a thread to poll and output results. ScheduledExecutorService pollingExecutor = Executors.newSingleThreadScheduledExecutor(); @@ -514,32 +553,36 @@ private Map runTest(Runnable whileRunning) new MessageTracker( numberOfMessages, cpsGcloudJavaPublisherCount + + kafkaMappedJavaPublisherCount + cpsGcloudPythonPublisherCount + cpsGcloudRubyPublisherCount + cpsGcloudGoPublisherCount); controller.startClients(messageTracker); + if (whileRunning != null) { whileRunning.run(); } + // Wait for the load test to finish. controller.waitForClients(); + statsMap = controller.getStatsForAllClientTypes(); printStats(statsMap); - Iterable missing = messageTracker.getMissing(); - if (missing.iterator().hasNext()) { - log.error("Some published messages were not received!"); - for (MessageIdentifier identifier : missing) { - log.error( - String.format( - "%d:%d", identifier.getPublisherClientId(), identifier.getSequenceNumber())); - } - } + if (spreadsheetId.length() > 0) { // Output results to common Google sheet new SheetsService(dataStoreDirectory, controller.getTypes()) .sendToSheets(spreadsheetId, statsMap); } + Iterable missing = messageTracker.getMissing(); + if (missing.iterator().hasNext()) { + log.error("Some published messages were not received!"); + log.error("Number of missing messages = " + Iterables.size(missing)); + } else if (numberOfMessages > 0) { + log.info("All sent messages were received successfully."); + log.info("Number of received messages = " + messageTracker.getNumberReceivedMessages()); + } return statsMap; } @@ -634,6 +677,7 @@ private void runNumCoresTest( GnuPlot gnuPlot = new GnuPlot(); CsvOutput csv = new CsvOutput(); for (cores = 1; cores <= 16; cores *= 2) { + Client.cores = cores; controller = controllerFunction.apply(project, clientParamsMap); if (controller == null) { System.exit(1); @@ -664,6 +708,8 @@ private void printStats(Map results) { results.forEach( (type, stats) -> { log.info("Results for " + type + ":"); + log.info("Messages out of order #: " + stats.numOutOrderMsgs); + log.info("Messages out of order %: " + stats.outOrderMsgsPercent); log.info("50%: " + LatencyDistribution.getNthPercentile(stats.bucketValues, 50.0)); log.info("99%: " + LatencyDistribution.getNthPercentile(stats.bucketValues, 99.0)); log.info("99.9%: " + LatencyDistribution.getNthPercentile(stats.bucketValues, 99.9)); @@ -717,7 +763,7 @@ public void validate(String name, String value) { } } - /** A converter from {@link String} to {@link com.google.protobuf.Duration}. */ + /** A converter from {@link String} to {@link Duration}. */ public static class DurationConverter extends BaseConverter { public DurationConverter(String optionName) { @@ -778,5 +824,4 @@ public Duration convert(String value) { } } } -} - +} \ No newline at end of file diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java index 597ab002..ac83ca23 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Client.java @@ -26,12 +26,16 @@ import com.google.pubsub.flic.common.LoadtestGrpc; import com.google.pubsub.flic.common.LoadtestProto; import com.google.pubsub.flic.common.LoadtestProto.KafkaOptions; +import com.google.pubsub.flic.common.LoadtestProto.MessageIdentifier; import com.google.pubsub.flic.common.LoadtestProto.PubsubOptions; import com.google.pubsub.flic.common.LoadtestProto.StartRequest; import com.google.pubsub.flic.common.LoadtestProto.StartResponse; import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -46,27 +50,30 @@ public class Client { public static final String TOPIC_PREFIX = "cloud-pubsub-loadtest-"; private static final Logger log = LoggerFactory.getLogger(Client.class); private static final int PORT = 5000; + public static int cores; + public static int partitions; public static int messageSize; public static int requestRate; - public static Timestamp startTime; - public static Duration loadtestDuration; public static int publishBatchSize; - public static Duration publishBatchDuration; + public static int replicationFactor; public static int maxMessagesPerPull; + public static int numberOfMessages = 0; + public static int maxOutstandingRequests; + public static Timestamp startTime; public static Duration pollDuration; + public static Duration burnInDuration; + public static Duration loadtestDuration; + public static Duration publishBatchDuration; public static String broker; public static String zookeeperIpAddress; - public static int maxOutstandingRequests; - public static Duration burnInDuration; - public static int numberOfMessages = 0; - public static int replicationFactor; - public static int partitions; - private final ClientType clientType; - private final String networkAddress; - private final String project; + public static boolean orderTest; private final String topic; + private final String project; private final String subscription; + private final String networkAddress; + private final ClientType clientType; private final ScheduledExecutorService executorService; + private final List messagesReceived = new ArrayList<>(); private ClientStatus clientStatus; private Supplier stubFactory; private LoadtestGrpc.LoadtestStub stub; @@ -98,8 +105,8 @@ public Client( this.project = project; this.topic = TOPIC_PREFIX + getTopicSuffix(clientType); this.subscription = subscription; - this.executorService = executorService; this.stubFactory = stubFactory; + this.executorService = executorService; } public static String getTopicSuffix(ClientType clientType) { @@ -119,6 +126,9 @@ public static String getTopicSuffix(ClientType clientType) { case KAFKA_PUBLISHER: case KAFKA_SUBSCRIBER: return "kafka"; + case KAFKA_MAPPED_JAVA_PUBLISHER: + case KAFKA_MAPPED_JAVA_SUBSCRIBER: + return "mapped"; } return null; } @@ -150,6 +160,8 @@ long[] getBucketValues() { return bucketValues; } + List getMessagesReceived() { return messagesReceived; } + void start(MessageTracker messageTracker) throws Throwable { this.messageTracker = messageTracker; // Send a gRPC call to start the server @@ -175,6 +187,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_GO_SUBSCRIBER: case CPS_GCLOUD_PYTHON_SUBSCRIBER: case CPS_GCLOUD_RUBY_SUBSCRIBER: + case KAFKA_MAPPED_JAVA_SUBSCRIBER: requestBuilder.setPubsubOptions(PubsubOptions.newBuilder().setSubscription(subscription)); break; case KAFKA_PUBLISHER: @@ -196,6 +209,7 @@ void start(MessageTracker messageTracker) throws Throwable { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: + case KAFKA_MAPPED_JAVA_PUBLISHER: break; } StartRequest request = requestBuilder.build(); @@ -260,6 +274,11 @@ public void onNext(LoadtestProto.CheckResponse checkResponse) { clientStatus = ClientStatus.STOPPED; doneFuture.set(null); } + if (orderTest && !clientType.isPublisher()) { + synchronized (this) { + messagesReceived.addAll(checkResponse.getReceivedMessagesList()); + } + } messageTracker.addAllMessageIdentifiers(checkResponse.getReceivedMessagesList()); synchronized (this) { for (int i = 0; i < LatencyDistribution.LATENCY_BUCKETS.length; i++) { @@ -303,7 +322,9 @@ public enum ClientType { CPS_GCLOUD_GO_PUBLISHER, CPS_GCLOUD_GO_SUBSCRIBER, KAFKA_PUBLISHER, - KAFKA_SUBSCRIBER; + KAFKA_SUBSCRIBER, + KAFKA_MAPPED_JAVA_PUBLISHER, + KAFKA_MAPPED_JAVA_SUBSCRIBER; public boolean isCpsPublisher() { switch (this) { @@ -311,6 +332,7 @@ public boolean isCpsPublisher() { case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: + case KAFKA_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -333,6 +355,7 @@ public boolean isPublisher() { case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: case KAFKA_PUBLISHER: + case KAFKA_MAPPED_JAVA_PUBLISHER: return true; default: return false; @@ -343,6 +366,8 @@ public ClientType getSubscriberType() { switch (this) { case CPS_GCLOUD_JAVA_PUBLISHER: return CPS_GCLOUD_JAVA_SUBSCRIBER; + case KAFKA_MAPPED_JAVA_PUBLISHER: + return KAFKA_MAPPED_JAVA_SUBSCRIBER; case KAFKA_PUBLISHER: return KAFKA_SUBSCRIBER; case CPS_GCLOUD_GO_PUBLISHER: diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Controller.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Controller.java index db3c7702..cb96d1bf 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Controller.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/Controller.java @@ -16,19 +16,24 @@ package com.google.pubsub.flic.controllers; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.pubsub.flic.common.LatencyDistribution; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.pubsub.flic.common.LoadtestProto.MessageIdentifier; + +import java.util.Arrays; import java.util.Set; +import java.util.Map; +import java.util.List; +import java.util.HashMap; +import java.util.ArrayList; +import java.util.stream.LongStream; +import java.util.stream.Collectors; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; -import java.util.stream.Collectors; -import java.util.stream.LongStream; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,19 +113,54 @@ public void waitForPublisherClients() throws Throwable { */ private LoadtestStats getStatsForClientType(Client.ClientType type) { LoadtestStats stats = new LoadtestStats(); + List clientsOfType = clients.stream() .filter(c -> c.getClientType() == type).collect(Collectors.toList()); + stats.runningSeconds = clientsOfType.stream().mapToLong(Client::getRunningSeconds).max().getAsLong() - Client.burnInDuration.getSeconds(); + clientsOfType.stream().map(Client::getBucketValues).forEach(bucketValues -> { for (int i = 0; i < LatencyDistribution.LATENCY_BUCKETS.length; i++) { stats.bucketValues[i] += bucketValues[i]; } }); + + stats.numOutOrderMsgs = new ArrayList<>(); + stats.outOrderMsgsPercent = new ArrayList<>(); + if (!type.isPublisher() && Client.orderTest) { + clientsOfType.stream().map(Client::getMessagesReceived).forEach(list -> { + MessageIdentifier[] msgs = new MessageIdentifier[list.size()]; + list.toArray(msgs); + long outOfOrderMsgsCount = countOutOfOrder(msgs); + stats.numOutOrderMsgs.add(outOfOrderMsgsCount); + stats.outOrderMsgsPercent.add( + Double.valueOf(String.format("%.2f", (outOfOrderMsgsCount * 100.0) / msgs.length))); + }); + } + return stats; } + private Long countOutOfOrder(MessageIdentifier[] msgs) { + Map counters = new HashMap<>(); + Map latestPublishTimestamps = new HashMap<>(); + + Arrays.stream(msgs).forEach(msg -> { + if (latestPublishTimestamps.get(msg.getPublisherClientId()) == null) { + counters.put(msg.getPublisherClientId(), 0L); + latestPublishTimestamps.put(msg.getPublisherClientId(), 0L); + } else if (msg.getPublishTime() < latestPublishTimestamps.get(msg.getPublisherClientId())) { + counters.put(msg.getPublisherClientId(), counters.get(msg.getPublisherClientId()) + 1); + } else { + latestPublishTimestamps.put(msg.getPublisherClientId(), msg.getPublishTime()); + } + }); + + return counters.values().stream().mapToLong(Number::longValue).sum(); + } + /** * Gets the results for all available types. * @@ -179,6 +219,8 @@ public void startClients(MessageTracker messageTracker) { /** The statistics that are exported by each load test client. */ public static class LoadtestStats { public long runningSeconds; + public List numOutOrderMsgs; + public List outOrderMsgsPercent; public long[] bucketValues = new long[LatencyDistribution.LATENCY_BUCKETS.length]; /** Returns the average QPS. */ public double getQPS() { diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java index 3654640d..171de28d 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java @@ -59,10 +59,12 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; +import kafka.utils.ZKStringSerializer; +import kafka.utils.ZkUtils; import kafka.admin.AdminUtils; import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils; import org.apache.commons.codec.digest.DigestUtils; import org.I0Itec.zkclient.ZkClient; import org.I0Itec.zkclient.ZkConnection; @@ -72,7 +74,7 @@ * This is a subclass of {@link Controller} that controls load tests on Google Compute Engine. */ public class GCEController extends Controller { - private static final String MACHINE_TYPE = "n1-standard-"; // standard machine prefix + private static final String MACHINE_TYPE = "custom-"; // standard machine prefix private static final String SOURCE_FAMILY = "projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20160930"; // Ubuntu 16.04 LTS private static final int ALREADY_EXISTS = 409; @@ -91,11 +93,19 @@ private GCEController(String projectName, Map int cores, ScheduledExecutorService executor, Storage storage, Compute compute, Pubsub pubsub) throws Throwable { super(executor); - this.projectName = projectName; this.types = types; this.cores = cores; this.storage = storage; this.compute = compute; + this.projectName = projectName; + + try { + createFirewall(); + createStorageBucket(); + } catch (Exception e) { + shutdown(e); + throw e; + } // For each unique type of CPS Publisher, create a Topic if it does not already exist, and then // delete and recreate any subscriptions attached to it so that we do not have backlog from @@ -131,11 +141,11 @@ private GCEController(String projectName, Map } catch (IOException e) { log.debug("Error deleting subscription, assuming it has not yet been created.", e); } + //TODO: fix the AckDeadline - Return it back to 10 instead of 600. try { pubsub.projects().subscriptions().create("projects/" + projectName + "/subscriptions/" + subscription, new Subscription() - .setTopic("projects/" + projectName + "/topics/" + topic) - .setAckDeadlineSeconds(10)).execute(); + .setTopic("projects/" + projectName + "/topics/" + topic)).execute(); } catch (IOException e) { pubsubFuture.setException(e); } @@ -155,11 +165,19 @@ private GCEController(String projectName, Map SettableFuture kafkaFuture = SettableFuture.create(); kafkaFutures.add(kafkaFuture); executor.execute(() -> { + try { + Thread.sleep(45000); + } catch (InterruptedException e) { } + String topic = Client.TOPIC_PREFIX + Client.getTopicSuffix(clientType); - ZkClient zookeeperClient = new ZkClient(Client.zookeeperIpAddress, 15000, 10000, - ZKStringSerializer$.MODULE$); + + ZkClient zookeeperClient = + new ZkClient(Client.zookeeperIpAddress, 30000, 5000, + ZKStringSerializer$.MODULE$); + ZkUtils zookeeperUtils = new ZkUtils(zookeeperClient, new ZkConnection(Client.zookeeperIpAddress), false); + try { if (AdminUtils.topicExists(zookeeperUtils, topic)) { log.info("Deleting topic " + topic + "."); @@ -176,11 +194,9 @@ private GCEController(String projectName, Map while (AdminUtils.topicExists(zookeeperUtils, topic)) { // waiting for topic to delete before recreating } - Properties topicConfig = new Properties(); AdminUtils .createTopic(zookeeperUtils, topic, Client.partitions, Client.replicationFactor, - AdminUtils.createTopic$default$5(), - AdminUtils.createTopic$default$6()); + AdminUtils.createTopic$default$5(), AdminUtils.createTopic$default$6()); log.info("Created topic " + topic + "."); } catch (Exception e) { kafkaFuture.setException(e); @@ -190,6 +206,7 @@ private GCEController(String projectName, Map zookeeperClient.close(); } } + kafkaFuture.set(null); }); }); @@ -197,9 +214,6 @@ private GCEController(String projectName, Map } try { - createStorageBucket(); - createFirewall(); - List> filesRemaining = new ArrayList<>(); Files.walk(Paths.get(resourceDirectory)) .filter(Files::isRegularFile).forEach(filePath -> { @@ -369,7 +383,7 @@ private void createStorageBucket() throws IOException { } } - /** + /** * Adds a firewall rule to the default network so that we can connect to our clients externally. */ private void createFirewall() throws IOException { @@ -379,7 +393,10 @@ private void createFirewall() throws IOException { .setAllowed(ImmutableList.of( new Firewall.Allowed() .setIPProtocol("tcp") - .setPorts(Collections.singletonList("5000")))); + .setPorts(Collections.singletonList("5000")), + new Firewall.Allowed() + .setIPProtocol("tcp") + .setPorts(Collections.singletonList("2181")))); try { compute.firewalls().insert(projectName, firewallRule).execute(); } catch (GoogleJsonResponseException e) { @@ -540,7 +557,7 @@ private InstanceTemplate defaultInstanceTemplate(String type) { return new InstanceTemplate() .setName("cps-loadtest-" + type + "-" + cores) .setProperties(new InstanceProperties() - .setMachineType(MACHINE_TYPE + cores) + .setMachineType(MACHINE_TYPE + cores + "-" + cores * 6144) .setDisks(Collections.singletonList(new AttachedDisk() .setBoot(true) .setAutoDelete(true) diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java index f387d27c..bb201da0 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java @@ -16,19 +16,19 @@ package com.google.pubsub.flic.controllers; -import com.google.pubsub.flic.common.LoadtestProto.MessageIdentifier; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; import java.util.Set; +import java.util.Map; +import java.util.HashSet; +import java.util.HashMap; +import com.google.pubsub.flic.common.LoadtestProto.MessageIdentifier; /** Ensures that no message loss has occurred. */ public class MessageTracker { - Map> receivedMessages = new HashMap<>(); - Set duplicates = new HashSet<>(); - final int numMessagesPerPublisher; final int numPublishers; + final int numMessagesPerPublisher; + Set duplicates = new HashSet<>(); + Map> receivedMessages = new HashMap<>(); public MessageTracker(int numMessagesPerPublisher, int numPublishers) { this.numMessagesPerPublisher = numMessagesPerPublisher; @@ -38,9 +38,14 @@ public MessageTracker(int numMessagesPerPublisher, int numPublishers) { synchronized void addAllMessageIdentifiers(Iterable identifiers) { identifiers.forEach( identifier -> { - receivedMessages.putIfAbsent(identifier.getPublisherClientId(), - new HashSet()); - if (!receivedMessages.get(identifier.getPublisherClientId()).add(identifier)) { + receivedMessages.putIfAbsent( + identifier.getPublisherClientId(), new HashSet()); + if (!receivedMessages.get( + identifier.getPublisherClientId()).add( + MessageIdentifier.newBuilder() + .setPublisherClientId(identifier.getPublisherClientId()) + .setSequenceNumber(identifier.getSequenceNumber()) + .build())) { duplicates.add(identifier); } }); @@ -50,6 +55,14 @@ synchronized Set getDuplicates() { return duplicates; } + public synchronized long getNumberReceivedMessages() { + long counter = 0; + for (Set set : receivedMessages.values()) { + counter += set.size(); + } + return counter; + } + public synchronized Iterable getMissing() { Set missing = new HashSet<>(); receivedMessages.forEach( @@ -73,4 +86,4 @@ public synchronized Iterable getMissing() { } return missing; } -} +} \ No newline at end of file diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 0f5c2a99..8caf7183 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -61,6 +61,8 @@ public class SheetsService { private int cpsSubscriberCount = 0; private int kafkaPublisherCount = 0; private int kafkaSubscriberCount = 0; + private int mappedPublisherCount = 0; + private int mappedSubscriberCount = 0; private String dataStoreDirectory; public SheetsService(String dataStoreDirectory, Map> types) { @@ -73,16 +75,24 @@ private void fillClientCounts(Map> types) { types.values().forEach(paramsMap -> { Map countMap = paramsMap.keySet().stream(). collect(Collectors.groupingBy( - ClientParams::getClientType, Collectors.summingInt(t -> 1))); + ClientParams::getClientType, Collectors.summingInt(t -> paramsMap.get(t)))); countMap.forEach((k, v) -> { if (k.isCpsPublisher()) { - cpsPublisherCount += v; + if (k.toString().startsWith("kafka")) { + mappedPublisherCount += v; + } else { + cpsPublisherCount += v; + } } else if (k.isKafkaPublisher()) { kafkaPublisherCount += v; - } else if (k.toString().startsWith("kafka")) { + } else if (k.toString().startsWith("kafka") && !k.toString().startsWith("kafka-mapped")) { kafkaSubscriberCount += v; } else { - cpsSubscriberCount += v; + if (k.toString().startsWith("kafka")) { + mappedSubscriberCount += v; + } else { + cpsSubscriberCount += v; + } } }); }); @@ -135,34 +145,52 @@ public List>> getValuesList(Map> kafkaValues = new ArrayList<>(results.size()); results.forEach((type, stats) -> { - List valueRow = new ArrayList<>(13); + List valueRow = new ArrayList<>(16); + valueRow.add(Client.cores); switch (type) { case CPS_GCLOUD_JAVA_PUBLISHER: case CPS_GCLOUD_PYTHON_PUBLISHER: case CPS_GCLOUD_RUBY_PUBLISHER: case CPS_GCLOUD_GO_PUBLISHER: - if (cpsPublisherCount == 0) { + case KAFKA_MAPPED_JAVA_PUBLISHER: + if (cpsPublisherCount == 0 && mappedPublisherCount == 0) { return; } - valueRow.add(cpsPublisherCount); - valueRow.add(0); + if (type.toString().startsWith("kafka")) { + valueRow.add("mapped"); + valueRow.add(mappedPublisherCount); + valueRow.add(0); + } else { + valueRow.add("cps"); + valueRow.add(cpsPublisherCount); + valueRow.add(0); + } cpsValues.add(0, valueRow); break; case CPS_GCLOUD_JAVA_SUBSCRIBER: case CPS_GCLOUD_GO_SUBSCRIBER: case CPS_GCLOUD_PYTHON_SUBSCRIBER: case CPS_GCLOUD_RUBY_SUBSCRIBER: - if (cpsSubscriberCount == 0) { + case KAFKA_MAPPED_JAVA_SUBSCRIBER: + if (cpsSubscriberCount == 0 && mappedSubscriberCount == 0) { return; } - valueRow.add(0); - valueRow.add(cpsSubscriberCount); + if (type.toString().startsWith("kafka")) { + valueRow.add("mapped"); + valueRow.add(0); + valueRow.add(mappedSubscriberCount); + } else { + valueRow.add("cps"); + valueRow.add(0); + valueRow.add(cpsSubscriberCount); + } cpsValues.add(valueRow); break; case KAFKA_PUBLISHER: if (kafkaPublisherCount == 0) { return; } + valueRow.add("kafka"); valueRow.add(kafkaPublisherCount); valueRow.add(0); kafkaValues.add(0, valueRow); @@ -171,6 +199,7 @@ public List>> getValuesList(Map>> getValuesList(Map> types = new HashMap<>(); int expectedCpsCount = 0; int expectedKafkaCount = 0; + int expectedMappedCount = 0; Map paramsMap = new HashMap<>(); for (ClientType type : ClientType.values()) { paramsMap.put(new ClientParams(type, ""), 1); if (type.toString().startsWith("cps")) { expectedCpsCount++; + } else if (type.toString().startsWith("kafka-mapped")) { + expectedMappedCount++; } else if (type.toString().startsWith("kafka")) { expectedKafkaCount++; } else { @@ -55,6 +58,8 @@ public void testClientSwitch() { service.getCpsPublisherCount() + service.getCpsSubscriberCount(), expectedCpsCount); assertEquals( service.getKafkaPublisherCount() + service.getKafkaSubscriberCount(), expectedKafkaCount); + assertEquals( + service.getMappedPublisherCount() + service.getMappedSubscriberCount(), expectedMappedCount); Map stats = new HashMap<>(); for (ClientType type : ClientType.values()) { diff --git a/pubsub-mapped-api/pom.xml b/pubsub-mapped-api/pom.xml index 5c99f4fd..25ea3ced 100644 --- a/pubsub-mapped-api/pom.xml +++ b/pubsub-mapped-api/pom.xml @@ -19,9 +19,6 @@ MappedApi http://maven.apache.org - - 1.3.0 - @@ -31,27 +28,27 @@ io.grpc grpc-netty - ${grpc.version} + 1.3.0 io.grpc grpc-auth - ${grpc.version} + 1.3.0 io.grpc grpc-protobuf - ${grpc.version} + 1.3.0 io.grpc grpc-stub - ${grpc.version} + 1.3.0 io.grpc grpc-testing - ${grpc.version} + 1.3.0 test @@ -144,7 +141,7 @@ com.google.guava guava - 23.0 + 22.0 diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java index 74b50945..4dbb7c95 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/Config.java @@ -15,6 +15,7 @@ package com.google.pubsub.clients.consumer; import com.google.common.base.Preconditions; +import com.google.pubsub.clients.producer.PubSubProducerConfig; import java.util.List; import java.util.Map; import java.util.Properties; @@ -27,10 +28,11 @@ class Config { - private final Boolean allowSubscriptionCreation; - private final Boolean allowSubscriptionDeletion; + private final String project; private final String groupId; private final int maxPollRecords; + private final Boolean allowSubscriptionCreation; + private final Boolean allowSubscriptionDeletion; private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; @@ -44,7 +46,6 @@ class Config { private final ConsumerInterceptors interceptors; private static final AtomicInteger CLIENT_ID = new AtomicInteger(1); - Config(Map configs) { this(ConsumerConfigCreator.getConsumerConfig(configs), new PubSubConsumerConfig(configs), @@ -119,6 +120,8 @@ private Config(ConsumerConfig consumerConfig, this.requestTimeoutMs = consumerConfig.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); //PubSub-specific options + + this.project = pubSubConsumerConfig.getString(PubSubProducerConfig.PROJECT_CONFIG); this.allowSubscriptionCreation = pubSubConsumerConfig.getBoolean(PubSubConsumerConfig.SUBSCRIPTION_ALLOW_CREATE_CONFIG); this.allowSubscriptionDeletion = @@ -128,13 +131,16 @@ private Config(ConsumerConfig consumerConfig, PubSubConsumerConfig.CREATED_SUBSCRIPTION_DEADLINE_SECONDS_CONFIG); this.maxAckExtensionPeriod = pubSubConsumerConfig.getInt(PubSubConsumerConfig.MAX_ACK_EXTENSION_PERIOD_SECONDS_CONFIG); - Preconditions.checkNotNull(this.allowSubscriptionCreation); Preconditions.checkNotNull(this.allowSubscriptionDeletion); Preconditions.checkNotNull(this.groupId); Preconditions.checkArgument(!this.groupId.isEmpty()); } + String getProject() { + return project; + } + Boolean getAllowSubscriptionCreation() { return allowSubscriptionCreation; } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index ca1837c0..7ad9f38c 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -112,10 +112,10 @@ public class KafkaConsumer implements Consumer { private static final ConsumerRebalanceListener NO_REBALANCE_LISTENER = null; private static final Logger log = LoggerFactory.getLogger(KafkaConsumer.class); - private static final String GOOGLE_CLOUD_PROJECT = "projects/" + - System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String SUBSCRIPTION_PREFIX = GOOGLE_CLOUD_PROJECT + "/subscriptions/"; - private static final String TOPIC_PREFIX = GOOGLE_CLOUD_PROJECT + "/topics/"; + + private final String TOPIC_PREFIX; + private final String SUBSCRIPTION_PREFIX; + private final String GOOGLE_CLOUD_PROJECT; //because of no partition concept, the partition number is constant private static final int DEFAULT_PARTITION = 0; @@ -175,15 +175,24 @@ private KafkaConsumer(Config configOptions) { try { log.debug("Starting PubSub subscriber"); + this.config = config; + this.GOOGLE_CLOUD_PROJECT = "projects/" + config.getProject(); + this.TOPIC_PREFIX = GOOGLE_CLOUD_PROJECT + "/topics/"; + this.SUBSCRIPTION_PREFIX = GOOGLE_CLOUD_PROJECT + "/subscriptions/"; + Preconditions.checkNotNull(channel); SubscriberFutureStub subscriberFutureStub = SubscriberGrpc.newFutureStub(channel) + .withMaxInboundMessageSize(16777216) + .withMaxOutboundMessageSize(16777216) .withDeadlineAfter(config.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); PublisherFutureStub publisherFutureStub = PublisherGrpc.newFutureStub(channel) + .withMaxInboundMessageSize(16777216) + .withMaxOutboundMessageSize(16777216) .withDeadlineAfter(config.getRequestTimeoutMs(), TimeUnit.MILLISECONDS); - if(callCredentials != null) { + if (callCredentials != null) { subscriberFutureStub = subscriberFutureStub.withCallCredentials(callCredentials); publisherFutureStub = publisherFutureStub.withCallCredentials(callCredentials); } @@ -191,8 +200,6 @@ private KafkaConsumer(Config configOptions) { this.subscriberFutureStub = subscriberFutureStub; this.publisherFutureStub = publisherFutureStub; - this.config = config; - log.debug("PubSub subscriber created"); } catch (Throwable t) { throw new KafkaException("Failed to construct PubSub subscriber", t); @@ -464,7 +471,7 @@ private void deleteSubscriptionsIfAllowed(Collection subscriptions * on next topic, as long as it polls some messages or gets though all topics. * * When polling, deadline extensions are scheduled. Messages deadlines will be extended until commit call - * (manual config), until auto.commit.interval passes since previous commit (auto config) + * (manual config), until auto.commit.interval passes since previous commit (auto config) * or until max.ack.extension.period passes (maximum on deadline extensions). * * If auto commit configured, every time poll is called it will check if auto.commit.interval passed since last @@ -549,11 +556,14 @@ private void checkPollPreconditions(long timeout) { } } + private long getMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000 + ts.getNanos() / 1000000; + } + private ConsumerRecord prepareKafkaRecord(ReceivedMessage receivedMessage, String topic) { PubsubMessage message = receivedMessage.getMessage(); - long timestamp = message.getPublishTime().getSeconds() * 1000 + message.getPublishTime().getNanos() / 1000; - + long timestamp = getMillis(message.getPublishTime()); TimestampType timestampType = TimestampType.CREATE_TIME; //because of no offset concept in PubSub, timestamp is treated as an offset @@ -943,7 +953,7 @@ public void onSuccess(@Nullable List empties) { @Override public void onFailure(Throwable throwable) { - //TODO retry unsubscribe? + //TODO: retry unsubscribe? log.warn("Failed to unsubscribe to topic", throwable); } } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java index 88ea8882..c305fc0d 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/PubSubConsumerConfig.java @@ -27,6 +27,10 @@ public class PubSubConsumerConfig extends AbstractConfig { private static final ConfigDef CONFIG = getInstance(); + /** project */ + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project that we will connect to."; + /** subscription.allow.create */ public static final String SUBSCRIPTION_ALLOW_CREATE_CONFIG = "subscription.allow.create"; private static final String SUBSCRIPTION_ALLOW_CREATE_DOC = @@ -53,6 +57,10 @@ public class PubSubConsumerConfig extends AbstractConfig { private static synchronized ConfigDef getInstance() { return new ConfigDef() + .define(PROJECT_CONFIG, + Type.STRING, + Importance.HIGH, + PROJECT_DOC) .define(SUBSCRIPTION_ALLOW_CREATE_CONFIG, Type.BOOLEAN, false, diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java index d97e4923..ed6e5d96 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/ack/Subscriber.java @@ -20,8 +20,8 @@ import com.google.api.core.ApiClock; import com.google.api.core.ApiService; import com.google.api.core.CurrentMillisClock; -import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.batching.FlowController; +import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.batching.FlowController.LimitExceededBehavior; import com.google.api.gax.core.Distribution; import com.google.api.gax.grpc.ExecutorProvider; @@ -108,10 +108,8 @@ private Subscriber(Builder builder) { this.nextCommitTime = clock.millisTime() + autoCommitIntervalMs; } - FlowController flowController = new FlowController( - builder - .flowControlSettings - .toBuilder() + FlowController flowController = + new FlowController(builder.flowControlSettings.toBuilder() .setLimitExceededBehavior(LimitExceededBehavior.ThrowException) .build()); @@ -211,7 +209,6 @@ public void commit(boolean sync) { commitBefore(sync, null); } - @Override public void doStart() { logger.log(Level.FINE, "Starting subscriber group."); @@ -250,14 +247,14 @@ private void stopAllPollingConnection() { /** Builder of {@link Subscriber Subscribers}. */ public static final class Builder { + private static final long DEFAULT_MEMORY_PERCENTAGE = 20; private static final Duration MIN_ACK_EXPIRATION_PADDING = Duration.ofMillis(100); private static final Duration DEFAULT_ACK_EXPIRATION_PADDING = Duration.ofMillis(500); - private static final long DEFAULT_MEMORY_PERCENTAGE = 20; MessageReceiver receiver; - Duration ackExpirationPadding = DEFAULT_ACK_EXPIRATION_PADDING; Duration maxAckExtensionPeriod; + Duration ackExpirationPadding = DEFAULT_ACK_EXPIRATION_PADDING; FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder() @@ -328,7 +325,7 @@ public Builder setSystemExecutorProvider(ExecutorProvider executorProvider) { this.systemExecutorProvider = Preconditions.checkNotNull(executorProvider); return this; } - + public Builder setAutoCommit(Boolean autoCommit) { this.autoCommit = autoCommit; return this; @@ -354,7 +351,7 @@ public Builder setSubscriberFutureStub(SubscriberFutureStub subscriberFutureStub this.subscriberFutureStub = subscriberFutureStub; return this; } - + public Builder setRetryBackoffMs(Long retryBackoffMs) { this.retryBackoffMs = retryBackoffMs; return this; diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java index 1f289a40..c8b7d710 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/Config.java @@ -16,63 +16,158 @@ package com.google.pubsub.clients.producer; +import java.util.List; import java.util.Map; -import com.google.common.annotations.VisibleForTesting; +import java.util.concurrent.atomic.AtomicInteger; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerConfigAdapter; +import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.common.config.ConfigDef.Range; -import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerInterceptor; +import org.apache.kafka.clients.producer.ProducerConfigCreator; +import org.apache.kafka.clients.producer.internals.ProducerInterceptors; /** - * Provides the configurations for a Kafka Producer instance. + * Provides the configurations for a Kafka Mapped Producer instance. */ -public class Config { +public class Config { + + private final String project; + private final long elementCount; + private final boolean autoCreate; + + private final int retries; + private final int timeout; + private final int batchSize; + private final long lingerMs; + private final long retriesMs; + private final long bufferMemorySize; + private final boolean acks; + private final String clientId; + + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final ProducerInterceptors interceptors; - private ProducerConfig kafkaConfigs; - private PubSubProducerConfig additionalConfigs; + private static final AtomicInteger CLIENT_ID = new AtomicInteger(1); - Config(Map configs) { + private final ProducerConfig kafkaConfigs; + private final PubSubProducerConfig additionalConfigs; + + Config(Map configs, Serializer keySerializer, Serializer valueSerializer) { additionalConfigs = new PubSubProducerConfig(configs); - kafkaConfigs = ProducerConfigAdapter.getConsumerConfig(configs); + + if (keySerializer == null && valueSerializer == null) { + kafkaConfigs = ProducerConfigCreator.getProducerConfig( + ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)); + } else { + kafkaConfigs = ProducerConfigCreator.getProducerConfig(configs); + } + + // CPS's configs + this.project = additionalConfigs.getString(PubSubProducerConfig.PROJECT_CONFIG); + this.elementCount = additionalConfigs.getLong(PubSubProducerConfig.ELEMENTS_COUNT_CONFIG); + this.autoCreate = additionalConfigs.getBoolean(PubSubProducerConfig.AUTO_CREATE_TOPICS_CONFIG); + + // Kafka's configs + if (keySerializer == null) { + this.keySerializer = kafkaConfigs.getConfiguredInstance( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); + + this.keySerializer.configure(kafkaConfigs.originals(), true); + } else { + kafkaConfigs.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); + this.keySerializer = keySerializer; + } + + if (valueSerializer == null) { + this.valueSerializer = kafkaConfigs.getConfiguredInstance( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); + + this.valueSerializer.configure(kafkaConfigs.originals(), false); + } else { + kafkaConfigs.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); + this.valueSerializer = valueSerializer; + } + + this.retries = kafkaConfigs.getInt(ProducerConfig.RETRIES_CONFIG); + this.timeout = kafkaConfigs.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.retriesMs = kafkaConfigs.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); + this.lingerMs = kafkaConfigs.getLong(ProducerConfig.LINGER_MS_CONFIG); + this.batchSize = kafkaConfigs.getInt(ProducerConfig.BATCH_SIZE_CONFIG); + this.acks = kafkaConfigs.getString(ProducerConfig.ACKS_CONFIG).matches("(-)?1|all"); + this.bufferMemorySize = kafkaConfigs.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); + + String id = kafkaConfigs.getString(ProducerConfig.CLIENT_ID_CONFIG); + if (id.length() <= 0) { + this.clientId = "producer-" + CLIENT_ID.getAndIncrement(); + } else { + this.clientId = id; + } + + kafkaConfigs.originals().put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + List> interceptorList = + (List) (ProducerConfigCreator + .getProducerConfig(kafkaConfigs.originals())) + .getConfiguredInstances( + ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, ProducerInterceptor.class); + this.interceptors = + interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); } - ProducerConfig getKafkaConfigs() { - return kafkaConfigs; + String getProject() { + return project; } - PubSubProducerConfig getAdditionalConfigs() { - return additionalConfigs; + boolean getAutoCreate() { + return autoCreate; } - public static class PubSubProducerConfig extends AbstractConfig { + long getElementCount() { + return elementCount; + } - public static final String PROJECT_CONFIG = "project"; - private static final String PROJECT_DOC = "GCP project that we will connect to."; + int getBatchSize() { + return batchSize; + } - public static final String ELEMENTS_COUNT_CONFIG = "element.count"; - private static final String ELEMENTS_COUNT_DOC = "This configuration controls the default count of" - + " elements in a batch."; + long getLingerMs() { + return lingerMs; + } - public static final String AUTO_CREATE_TOPICS_CONFIG = "auto.create.topics.enable"; - private static final String AUTO_CREATE_TOPICS_DOC = "When true topics are automatically created" - + " if they don't exist."; + long getBufferMemorySize() { + return bufferMemorySize; + } - private static final ConfigDef CONFIG = new ConfigDef() - .define(PROJECT_CONFIG, Type.STRING, Importance.HIGH, PROJECT_DOC) - .define(AUTO_CREATE_TOPICS_CONFIG, Type.BOOLEAN, true, Importance.MEDIUM, AUTO_CREATE_TOPICS_DOC) - .define(ELEMENTS_COUNT_CONFIG, Type.LONG, 1000L, Range.atLeast(1L), Importance.MEDIUM, ELEMENTS_COUNT_DOC); + int getRetries() { + return retries; + } - PubSubProducerConfig(Map originals, boolean doLog) { - super(CONFIG, originals, doLog); - } + long getRetriesMs() { + return retriesMs; + } - PubSubProducerConfig(Map originals) { - super(CONFIG, originals); - } + int getTimeout() { + return timeout; + } + + ProducerInterceptors getInterceptors() { + return interceptors; + } + + Serializer getKeySerializer() { + return keySerializer; + } + + Serializer getValueSerializer() { + return valueSerializer; + } + + boolean getAcks() { + return acks; + } + + String getClientId() { + return clientId; } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java index 0e0e9ffc..0ef107e9 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java @@ -32,7 +32,6 @@ import com.google.protobuf.ByteString; import com.google.pubsub.v1.TopicName; import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.clients.producer.Config.PubSubProducerConfig; import java.io.IOException; @@ -60,121 +59,49 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerInterceptor; -import org.apache.kafka.clients.producer.ProducerConfigAdapter; -import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; /** * A Kafka client that publishes records to Google Cloud Pub/Sub. */ public class KafkaProducer implements Producer { + private final AtomicBoolean closed; + private static final long VERSION = 1L; private static final double MULTIPLIER = 1.0; - private static final AtomicInteger CLIENT_ID = new AtomicInteger(1); private Config producerConfig; private Map publishers; - private final String project; - private final Serializer keySerializer; - private final Serializer valueSerializer; - - private final int retries; - private final int timeout; - private final int batchSize; - private final long lingerMs; - private final long retriesMs; - private final long elementCount; - private final long bufferMemorySize; - private final boolean isAcks; - private final boolean autoCreate; - private final String clientId; - private final ProducerInterceptors interceptors; - - private final AtomicBoolean closed; - public KafkaProducer(Map configs) { - this(new Config(configs), null, null); + this(new Config(configs, null, null)); } public KafkaProducer(Map configs, Serializer keySerializer, Serializer valueSerializer) { - this(new Config(ProducerConfig.addSerializerToConfig( - configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); + this(new Config(configs, keySerializer, valueSerializer)); } public KafkaProducer(Properties properties) { - this(new Config(properties), null, null); + this(new Config(properties, null, null)); } public KafkaProducer(Properties properties, Serializer keySerializer, Serializer valueSerializer) { - this(new Config(ProducerConfig.addSerializerToConfig( - properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); + this(new Config(properties, keySerializer, valueSerializer)); } @VisibleForTesting @SuppressWarnings("unchecked") - public KafkaProducer(Config configs, Serializer keySerializer, Serializer valueSerializer) { - + public KafkaProducer(Config configs) { producerConfig = configs; - // CPS's configs - project = configs.getAdditionalConfigs().getString(PubSubProducerConfig.PROJECT_CONFIG); - autoCreate = configs.getAdditionalConfigs().getBoolean(PubSubProducerConfig.AUTO_CREATE_TOPICS_CONFIG); - elementCount = configs.getAdditionalConfigs().getLong(PubSubProducerConfig.ELEMENTS_COUNT_CONFIG); - - // Kafka's configs - if (keySerializer == null) { - this.keySerializer = configs.getKafkaConfigs().getConfiguredInstance( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); - - this.keySerializer.configure(configs.getKafkaConfigs().originals(), true); - } else { - configs.getKafkaConfigs().ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); - this.keySerializer = keySerializer; - } - - if (valueSerializer == null) { - this.valueSerializer = configs.getKafkaConfigs().getConfiguredInstance( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class); - - this.valueSerializer.configure(configs.getKafkaConfigs().originals(), false); - } else { - configs.getKafkaConfigs().ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); - this.valueSerializer = valueSerializer; - } - closed = new AtomicBoolean(false); - retries = configs.getKafkaConfigs().getInt(ProducerConfig.RETRIES_CONFIG); - timeout = configs.getKafkaConfigs().getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); - retriesMs = configs.getKafkaConfigs().getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); - lingerMs = configs.getKafkaConfigs().getLong(ProducerConfig.LINGER_MS_CONFIG); - batchSize = configs.getKafkaConfigs().getInt(ProducerConfig.BATCH_SIZE_CONFIG); - isAcks = configs.getKafkaConfigs().getString(ProducerConfig.ACKS_CONFIG).matches("(-)?1|all"); - bufferMemorySize = configs.getKafkaConfigs().getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); - - String Id = configs.getKafkaConfigs().getString(ProducerConfig.CLIENT_ID_CONFIG); - if (Id.length() <= 0) { - clientId = "producer-" + CLIENT_ID.getAndIncrement(); - } else { - clientId = Id; - } - configs.getKafkaConfigs().originals().put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - List> interceptorList = - (List) (ProducerConfigAdapter.getConsumerConfig(configs.getKafkaConfigs().originals())).getConfiguredInstances( - ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, ProducerInterceptor.class); - this.interceptors = - interceptorList.isEmpty() ? null : new ProducerInterceptors<>(interceptorList); - publishers = new ConcurrentHashMap<>(); } @@ -188,11 +115,11 @@ private Publisher createPublisher(String topic) { TopicAdminClient topicAdmin = null; try { - topicName = TopicName.create(project, topic); + topicName = TopicName.create(producerConfig.getProject(), topic); topicAdmin = TopicAdminClient.create(); topicAdmin.getTopic(topicName); } catch (Exception e) { - if (e.getMessage().contains("NOT_FOUND") && autoCreate) { + if (e.getMessage().contains("NOT_FOUND") && producerConfig.getAutoCreate()) { topicAdmin.createTopic(topicName); } else { throw new KafkaException("Failed to construct kafka producer, Topic not found.", e); @@ -203,27 +130,30 @@ private Publisher createPublisher(String topic) { pub = Publisher.defaultBuilder(topicName) .setBatchingSettings(BatchingSettings.newBuilder() - .setElementCountThreshold(elementCount) - .setRequestByteThreshold((long) batchSize) - .setDelayThreshold(Duration.ofMillis(lingerMs)) + .setElementCountThreshold(producerConfig.getElementCount()) + .setRequestByteThreshold((long) producerConfig.getBatchSize()) + .setDelayThreshold(Duration.ofMillis(producerConfig.getLingerMs())) .setFlowControlSettings(FlowControlSettings.newBuilder() - .setMaxOutstandingRequestBytes(bufferMemorySize) + .setMaxOutstandingRequestBytes(producerConfig.getBufferMemorySize()) .build()) .build()) .setRetrySettings(RetrySettings.newBuilder() - .setMaxAttempts(retries) + .setMaxAttempts(producerConfig.getRetries()) .setRetryDelayMultiplier(MULTIPLIER) - .setInitialRetryDelay(Duration.ofMillis(retriesMs)) - .setMaxRetryDelay(Duration.ofMillis((retries + 1) * retriesMs)) + .setInitialRetryDelay(Duration.ofMillis(producerConfig.getRetriesMs())) + .setMaxRetryDelay(Duration.ofMillis( + (producerConfig.getRetries() + 1) * producerConfig.getRetriesMs())) .setRpcTimeoutMultiplier(MULTIPLIER) - .setInitialRpcTimeout(Duration.ofMillis(timeout)) - .setMaxRpcTimeout(Duration.ofMillis((retries + 1) * timeout)) + .setInitialRpcTimeout(Duration.ofMillis(producerConfig.getTimeout())) + .setMaxRpcTimeout(Duration.ofMillis( + (producerConfig.getRetries() + 1) * producerConfig.getTimeout())) - .setTotalTimeout(Duration.ofMillis((retries + 2) * timeout)) + .setTotalTimeout(Duration.ofMillis( + (producerConfig.getRetries() + 2) * producerConfig.getTimeout())) .build()) .build(); @@ -244,7 +174,6 @@ public Future send(ProducerRecord originalRecord) { /** * Sends the given record and invokes the specified callback. - * The given record must have the same topic as the producer. */ public Future send(ProducerRecord originalRecord, Callback callback) { if (closed.get()) { @@ -252,7 +181,8 @@ public Future send(ProducerRecord originalRecord, Callback } ProducerRecord record = - this.interceptors == null ? originalRecord : this.interceptors.onSend(originalRecord); + producerConfig.getInterceptors() == + null ? originalRecord : producerConfig.getInterceptors().onSend(originalRecord); if (record == null) { throw new NullPointerException(); @@ -263,11 +193,11 @@ public Future send(ProducerRecord originalRecord, Callback byte[] valueBytes = null; if (record.value() != null) { try { - valueBytes = valueSerializer.serialize(record.topic(), record.value()); + valueBytes = producerConfig.getValueSerializer().serialize(record.topic(), record.value()); } catch (ClassCastException e) { throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + " to class " + - producerConfig.getKafkaConfigs().getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + + producerConfig.getValueSerializer().getClass().getName() + " specified in value.serializer"); } } @@ -279,16 +209,16 @@ public Future send(ProducerRecord originalRecord, Callback byte[] keyBytes = null; if (record.key() != null) { try { - keyBytes = keySerializer.serialize(record.topic(), record.key()); + keyBytes = producerConfig.getKeySerializer().serialize(record.topic(), record.key()); } catch (ClassCastException cce) { throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + " to class " + - producerConfig.getKafkaConfigs().getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + + producerConfig.getKeySerializer().getClass().getName() + " specified in key.serializer"); } } - PubsubMessage msg = createMessage(record, dateTime.getMillis(), keyBytes, valueBytes); + PubsubMessage msg = createMessage(dateTime.getMillis(), keyBytes, valueBytes); ApiFuture messageIdFuture = publishers.computeIfAbsent( record.topic(), topic -> createPublisher(topic)).publish(msg); @@ -300,7 +230,7 @@ public Future send(ProducerRecord originalRecord, Callback final SettableFuture future = SettableFuture.create(); if (callback != null) { - if (isAcks) { + if (producerConfig.getAcks()) { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { private RecordMetadata recordMetadata = getRecordMetadata(topic, dateTime.getMillis(), keySize, valueSize); @@ -329,10 +259,10 @@ public void onSuccess(String result) { } // Attribute's value's size shouldn't exceed 1024 bytes (256 for key) - private PubsubMessage createMessage(ProducerRecord record, Long dateTime, byte[] key, byte[] value) { + private PubsubMessage createMessage(Long dateTime, byte[] key, byte[] value) { Map attributes = new HashMap<>(); - attributes.put("id", clientId); + attributes.put("id", producerConfig.getClientId()); attributes.put("offset", Long.toString(dateTime)); @@ -344,12 +274,13 @@ private PubsubMessage createMessage(ProducerRecord record, Long dateTime, throw new SerializationException("Key size should be at most 1024 bytes."); } - return PubsubMessage.newBuilder().setData(ByteString.copyFrom(value)).putAllAttributes(attributes).build(); + return PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(value)).putAllAttributes(attributes).build(); } private void callbackOnCompletion(Callback cb, RecordMetadata m, Exception e) { - if (interceptors != null) { - interceptors.onAcknowledgement(m, e); + if (producerConfig.getInterceptors() != null) { + producerConfig.getInterceptors().onAcknowledgement(m, e); } cb.onCompletion(m, e); } @@ -371,7 +302,7 @@ public void flush() { pub.getValue().shutdown(); } } catch (Exception e) { - throw new InterruptException("Flush interrupted.", (InterruptedException) e); + throw new InterruptException("Flush interrupted."); } } @@ -411,12 +342,12 @@ public void close(long timeout, TimeUnit unit) { pub.getValue().shutdown(); } - if (interceptors != null) { - interceptors.close(); + if (producerConfig.getInterceptors() != null) { + producerConfig.getInterceptors().close(); } - keySerializer.close(); - valueSerializer.close(); + producerConfig.getKeySerializer().close(); + producerConfig.getValueSerializer().close(); } catch (Exception e) { throw new KafkaException("Failed to close kafka producer", e); } diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java new file mode 100644 index 00000000..9e73b120 --- /dev/null +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java @@ -0,0 +1,51 @@ +package com.google.pubsub.clients.producer; + +import java.util.Map; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigDef.Range; +import org.apache.kafka.common.config.ConfigDef.Importance; + +/** + * The Producer configuration keys + */ +public class PubSubProducerConfig extends AbstractConfig { + + public static final String PROJECT_CONFIG = "project"; + private static final String PROJECT_DOC = "GCP project that we will connect to."; + + public static final String ELEMENTS_COUNT_CONFIG = "elements.count"; + private static final String ELEMENTS_COUNT_DOC = "This configuration controls the default count of" + + " elements in a batch."; + + public static final String AUTO_CREATE_TOPICS_CONFIG = "auto.create.topics.enable"; + private static final String AUTO_CREATE_TOPICS_DOC = "When true topics are automatically created" + + " if they don't exist."; + + private static final ConfigDef CONFIG = new ConfigDef() + .define(PROJECT_CONFIG, + Type.STRING, + Importance.HIGH, + PROJECT_DOC) + .define(AUTO_CREATE_TOPICS_CONFIG, + Type.BOOLEAN, + true, + Importance.MEDIUM, + AUTO_CREATE_TOPICS_DOC) + .define(ELEMENTS_COUNT_CONFIG, + Type.LONG, + 1000L, + Range.atLeast(1L), + Importance.MEDIUM, + ELEMENTS_COUNT_DOC); + + PubSubProducerConfig(Map originals, boolean doLog) { + super(CONFIG, originals, doLog); + } + + PubSubProducerConfig(Map originals) { + super(CONFIG, originals); + } +} diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java index a7819428..0e846804 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/common/ChannelUtil.java @@ -18,7 +18,8 @@ public final class ChannelUtil { private ChannelUtil() throws IOException { this.callCredentials = MoreCallCredentials.from(GoogleCredentials.getApplicationDefault()); - this.channel = ManagedChannelBuilder.forTarget(ENDPOINT).build(); + this.channel = + ManagedChannelBuilder.forTarget(ENDPOINT).build(); } private static synchronized ChannelUtil buildInstance() { diff --git a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java index 0a39446a..3b620e57 100644 --- a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java +++ b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfigCreator.java @@ -36,6 +36,6 @@ public static ConsumerConfig getConsumerConfig(Map properties) { private static void addDefaultKafkaRequiredOptions(Map map) { if(!map.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) - map.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:8000"); + map.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); } } diff --git a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java similarity index 71% rename from pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java rename to pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java index 2541a1c5..ca20d837 100644 --- a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigAdapter.java +++ b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java @@ -22,24 +22,25 @@ /** * This is an adapter to control the ProducerConfig class since it's constructors are package-private. */ -public class ProducerConfigAdapter { +public class ProducerConfigCreator { - public static ProducerConfig getConsumerConfig(Properties properties) { - return getConsumerConfig(properties); + public static ProducerConfig getProducerConfig(Properties properties) { + return getProducerConfig(properties); } - public static ProducerConfig getConsumerConfig(Map configurations) { + public static ProducerConfig getProducerConfig(Map configurations) { addDefaultKafkaRequiredConfigs(configurations); return new ProducerConfig(configurations); } private static void addDefaultKafkaRequiredConfigs(Map map) { if (!map.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) { - map.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:8080"); + map.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); } - if (!map.containsKey(ProducerConfig.LINGER_MS_CONFIG)) { - map.put(ProducerConfig.LINGER_MS_CONFIG, 1); + if (!map.containsKey(ProducerConfig.LINGER_MS_CONFIG) + || Integer.parseInt((String) map.get(ProducerConfig.LINGER_MS_CONFIG)) == 0) { + map.put(ProducerConfig.LINGER_MS_CONFIG, "1"); } } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java index d7341235..9733621c 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ConfigTest.java @@ -88,6 +88,7 @@ private Properties getProperties() { private ImmutableMap getTestOptionsMap() { return new ImmutableMap.Builder<>() + .put("project", "unit-test-project") .put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") .put("value.deserializer", diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java index a848aa62..484a76d1 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/KafkaConsumerTest.java @@ -458,6 +458,7 @@ public void interceptorsOnConsume() { Properties properties = new Properties(); properties.putAll(new ImmutableMap.Builder<>() + .put("project", "unit-test-project") .put("key.deserializer", "org.apache.kafka.common.serialization.IntegerDeserializer") .put("value.deserializer", @@ -482,15 +483,16 @@ public void interceptorsOnConsume() { } } - private KafkaConsumer getConsumer(boolean allowesCreation) { - Properties properties = getTestProperties(allowesCreation); + private KafkaConsumer getConsumer(boolean allowsCreation) { + Properties properties = getTestProperties(allowsCreation); Config configOptions = new Config(properties); return new KafkaConsumer<>(configOptions, grpcServerRule.getChannel(), null); } - private Properties getTestProperties(boolean allowesCreation) { + private Properties getTestProperties(boolean allowsCreation) { Properties properties = new Properties(); properties.putAll(new ImmutableMap.Builder<>() + .put("project", "unit-test-project") .put("key.deserializer", "org.apache.kafka.common.serialization.IntegerDeserializer") .put("value.deserializer", @@ -498,17 +500,13 @@ private Properties getTestProperties(boolean allowesCreation) { .put("max.poll.records", 500) .put("group.id", "groupId") .put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false) - .put("subscription.allow.create", allowesCreation) + .put("subscription.allow.create", allowsCreation) .put("subscription.allow.delete", false) .build() ); return properties; } - private String getTopicPrefixString(String topicName) { - return "projects/" + System.getenv("GOOGLE_CLOUD_PROJECT") + "/topics/" + topicName; - } - static class SubscriberGetImpl extends SubscriberImplBase { private Timestamp keptTimestamp = Timestamp.newBuilder().setSeconds(1500).build(); @@ -652,7 +650,7 @@ static class PublisherImpl extends PublisherImplBase { @Override public void listTopics(ListTopicsRequest request, StreamObserver responseObserver) { - String project = "projects/" + System.getenv("GOOGLE_CLOUD_PROJECT") + "/topics/"; + String project = "projects/unit-test-project/topics/"; List topicNames = new ArrayList<>(Arrays.asList( project + "thisis123cat", project + "abc12345bad", project + "noWay1234", project + "funnycats000cat")); diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java index 2311b60c..18424849 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/consumer/ack/SubscriberTest.java @@ -20,34 +20,41 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.grpc.FixedExecutorProvider; + import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.pubsub.clients.consumer.ack.FakeSubscriberServiceImpl.ModifyAckDeadline; + import com.google.pubsub.clients.consumer.ack.Subscriber.Builder; -import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.clients.consumer.ack.FakeSubscriberServiceImpl.ModifyAckDeadline; + import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import com.google.pubsub.v1.SubscriberGrpc; -import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.SubscriberGrpc; +import com.google.pubsub.v1.ReceivedMessage; import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.SubscriberGrpc.SubscriberFutureStub; + import io.grpc.ManagedChannel; +import io.grpc.internal.ServerImpl; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.ServerImpl; + import java.io.IOException; + +import java.util.List; import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; -import org.junit.After; -import org.junit.Before; + import org.junit.Rule; import org.junit.Test; +import org.junit.After; +import org.junit.Before; import org.junit.rules.TestName; import org.threeten.bp.Duration; diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java index 56ac78de..784fc556 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ConfigTest.java @@ -16,21 +16,18 @@ package com.google.pubsub.clients.producer; -import org.junit.Rule; -import org.junit.Test; -import org.junit.Assert; -import org.junit.rules.ExpectedException; - import java.util.Properties; import com.google.common.collect.ImmutableMap; -import com.google.pubsub.clients.producer.Config.PubSubProducerConfig; import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.Assert; +import org.junit.rules.ExpectedException; + public class ConfigTest { @Rule @@ -47,24 +44,22 @@ public void successAllConfigsProvided() { .build() ); - Config testConfig = new Config(props); + Config testConfig = new Config(props, null, null); Assert.assertEquals("Project config equals unit-test-project.", - "unit-test-project", testConfig.getAdditionalConfigs().getString(PubSubProducerConfig.PROJECT_CONFIG)); + "unit-test-project", testConfig.getProject()); Assert.assertNotNull( - "Key serializer must not be null.", testConfig.getKafkaConfigs() - .getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class)); + "Key serializer must not be null.", testConfig.getKeySerializer()); Assert.assertEquals( "Key serializer config must equal StringSerializer.", StringSerializer.class, - testConfig.getKafkaConfigs().getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)); + testConfig.getKeySerializer().getClass()); Assert.assertNotNull( - "Value serializer must not be null.", testConfig.getKafkaConfigs(). - getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class)); + "Value serializer must not be null.", testConfig.getValueSerializer()); Assert.assertEquals( "Value serializer config must equal StringSerializer.", StringSerializer.class, - testConfig.getKafkaConfigs().getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)); + testConfig.getValueSerializer().getClass()); } @Test @@ -78,7 +73,7 @@ public void noSerializerProvided() { exception.expect(ConfigException.class); - new Config(props); + new Config(props, null, null); } @Test @@ -92,6 +87,6 @@ public void noTopicProvided() { exception.expect(ConfigException.class); - new Config(props); + new Config(props, null, null); } } \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java index 5c5fb426..2b0288d4 100644 --- a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/KafkaProducerTest.java @@ -19,43 +19,51 @@ import static org.powermock.api.mockito.PowerMockito.when; import com.google.api.core.ApiFuture; -import com.google.api.gax.batching.BatchingSettings; import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.batching.BatchingSettings; + +import com.google.common.collect.ImmutableMap; + import com.google.cloud.pubsub.v1.Publisher; import com.google.cloud.pubsub.v1.Publisher.Builder; import com.google.cloud.pubsub.v1.TopicAdminClient; -import com.google.common.collect.ImmutableMap; -import com.google.pubsub.v1.PubsubMessage; + import com.google.pubsub.v1.TopicName; -import java.io.ByteArrayOutputStream; +import com.google.pubsub.v1.PubsubMessage; + import java.io.IOException; -import java.io.OutputStream; import java.io.PrintStream; -import java.util.ArrayList; +import java.io.OutputStream; +import java.io.ByteArrayOutputStream; + import java.util.List; -import java.util.Map; +import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.TimeUnit; + import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerInterceptor; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.IntegerDeserializer; -import org.apache.kafka.common.serialization.IntegerSerializer; + import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.Assert; -import org.junit.Before; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; + import org.junit.Test; +import org.junit.Before; +import org.junit.Assert; import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Matchers; + import org.mockito.Mockito; +import org.mockito.Matchers; +import org.mockito.ArgumentCaptor; + import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.core.classloader.annotations.PrepareForTest; @RunWith(PowerMockRunner.class) @PrepareForTest({Publisher.class, TopicAdminClient.class}) @@ -266,42 +274,4 @@ public void interceptRecordsWithException() { Assert.assertEquals(10 * key, Integer.parseInt(os.toString())); } - - public static class MultiplyByTenInterceptor implements ProducerInterceptor { - @Override - public ProducerRecord onSend(ProducerRecord producerRecord) { - int updatedValue = 10 * producerRecord.value(); - System.out.print(updatedValue); - return new ProducerRecord(producerRecord.topic(), producerRecord.key(), updatedValue); - } - - @Override - public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } - - @Override - public void close() { - System.out.print("Closed"); - } - - @Override - public void configure(Map map) { } - } - - public static class ThrowExceptionInterceptor implements ProducerInterceptor { - @Override - public ProducerRecord onSend(ProducerRecord producerRecord) { - throw new RuntimeException(); - } - - @Override - public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } - - @Override - public void close() { - System.out.print("Closed"); - } - - @Override - public void configure(Map map) { } - } -} +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/MultiplyByTenInterceptor.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/MultiplyByTenInterceptor.java new file mode 100644 index 00000000..9357a873 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/MultiplyByTenInterceptor.java @@ -0,0 +1,42 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import java.util.Map; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.ProducerInterceptor; + +public class MultiplyByTenInterceptor implements ProducerInterceptor { + @Override + public ProducerRecord onSend(ProducerRecord producerRecord) { + int updatedValue = 10 * producerRecord.value(); + System.out.print(updatedValue); + return new ProducerRecord(producerRecord.topic(), producerRecord.key(), updatedValue); + } + + @Override + public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } + + @Override + public void close() { + System.out.print("Closed"); + } + + @Override + public void configure(Map map) { } +} \ No newline at end of file diff --git a/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ThrowExceptionInterceptor.java b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ThrowExceptionInterceptor.java new file mode 100644 index 00000000..fac52424 --- /dev/null +++ b/pubsub-mapped-api/src/test/java/com/google/pubsub/clients/producer/ThrowExceptionInterceptor.java @@ -0,0 +1,40 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +package com.google.pubsub.clients.producer; + +import java.util.Map; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.ProducerInterceptor; + +public class ThrowExceptionInterceptor implements ProducerInterceptor { + @Override + public ProducerRecord onSend(ProducerRecord producerRecord) { + throw new RuntimeException(); + } + + @Override + public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) { } + + @Override + public void close() { + System.out.print("Closed"); + } + + @Override + public void configure(Map map) { } +} From e73d30836e195c1b7f7791a460766e0163966dd7 Mon Sep 17 00:00:00 2001 From: gewillovic Date: Fri, 13 Oct 2017 16:15:05 -0700 Subject: [PATCH 140/140] Java docs and comments - producer. (#134) --- load-test-framework/driver.py | 53 +++++++++++++++++ .../pubsub/clients/common/LoadTestRunner.java | 4 +- .../google/pubsub/clients/common/Task.java | 5 ++ .../clients/mapped/CPSSubscriberTask.java | 2 + .../java/com/google/pubsub/flic/Driver.java | 1 + .../flic/controllers/GCEController.java | 5 +- .../flic/controllers/MessageTracker.java | 1 + .../pubsub/flic/output/SheetsService.java | 2 + .../clients/consumer/KafkaConsumer.java | 1 - .../clients/producer/KafkaProducer.java | 58 ++++++++++++++++--- .../producer/PubSubProducerConfig.java | 3 + .../producer/ProducerConfigCreator.java | 2 + 12 files changed, 126 insertions(+), 11 deletions(-) diff --git a/load-test-framework/driver.py b/load-test-framework/driver.py index 28e39c79..8e3d8bb9 100755 --- a/load-test-framework/driver.py +++ b/load-test-framework/driver.py @@ -1,5 +1,19 @@ #!/usr/bin/env python +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import sys import getopt @@ -8,6 +22,45 @@ def main(project, test, vms_count, zone, sheet_id, broker, zookeeper, partitions, replication, mapped, cps): + """ + Runs the load test framework. + + Args: + project: The name of the Google Cloud project. + + test: The type of test to run. Valid options are 'latency', 'service', + 'throughput', 'test_throughput', 'ordering', 'correctness'. + + vms_count: The number of VMs to start for each client type. You must have + sufficient Google Compute Engine quota to start vms_count * + len(client_types) * 4 cores, and it will take 4 times as much + quota to run a 'throughput' or 'service' test. + + zone: the region to instantiate the instances in. + + sheet_id: the Google Sheets ID, needed to publish results to. + + broker: The network address of the Kafka broker to connect to. If supplied + we will automatically start Kafka clients. + + zookeeper: The address of the zookeeper cluster, we can create and manage topics + automatically if provided. + + partitions: The number of partitions in a single topic - Kafka config. + + replication: The replication factor for every topic - Kafka config. + + mapped: A flag to run the test for the mapped API or not. + + cps: A flag to run the test for CPS or not. + + Arguments Example: + + --project= --vms_count=<# of VMs> --test=correctness --mapped --cps + --sheet_id= --broker= + --zookeeper= + """ + subprocess.call(['mvn', 'clean', 'install'], cwd='../pubsub-mapped-api') subprocess.call(['mvn', 'clean', 'package']) diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java b/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java index 74dffac0..98763349 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/common/LoadTestRunner.java @@ -145,12 +145,12 @@ public void check( boolean finishedValue = finished.get(); responseObserver.onNext( CheckResponse.newBuilder() + .addAllReceivedMessages( + task.flushMessageIdentifiers(request.getDuplicatesList())) .addAllBucketValues(task.getBucketValues()) .setRunningDuration( Duration.newBuilder() .setSeconds(stopwatch.elapsed(TimeUnit.SECONDS))) - .addAllReceivedMessages( - task.flushMessageIdentifiers(request.getDuplicatesList())) .setIsFinished(finishedValue) .build()); responseObserver.onCompleted(); diff --git a/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java b/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java index 1682c239..1848164b 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java +++ b/load-test-framework/src/main/java/com/google/pubsub/clients/common/Task.java @@ -47,6 +47,8 @@ */ public abstract class Task implements Runnable { private final MetricsHandler metricsHandler; + + // Linked HashMap to maintain the order. private final Map identifiers = new LinkedHashMap<>(); private final Map identifiersToRecord = new LinkedHashMap<>(); private final long burnInTimeMillis; @@ -56,6 +58,7 @@ public abstract class Task implements Runnable { private final AtomicInteger numMessages = new AtomicInteger(0); private final AtomicLong lastUpdateMillis = new AtomicLong(System.currentTimeMillis()); + // Used to keep track of the count of sent messages, successful or not. protected final AtomicInteger actualCounter = new AtomicInteger(0); protected Task(StartRequest request, String type, MetricsHandler.MetricName metricName) { @@ -212,6 +215,8 @@ synchronized List flushMessageIdentifiers(List doRun() { return Futures.immediateFailedFuture( new IllegalStateException("The task is shut down.")); } + + // Record the publishTime set by the server, and the receiveTime at which it's recorded. records.forEach( record -> { String[] tokens = record.key().split("#"); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java index 35732eaa..2327c763 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/Driver.java @@ -575,6 +575,7 @@ private Map runTest(Runnable whileRunning) .sendToSheets(spreadsheetId, statsMap); } + // It's not that helpful to print all the messages, our main concern is the count. Iterable missing = messageTracker.getMissing(); if (missing.iterator().hasNext()) { log.error("Some published messages were not received!"); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java index 171de28d..68cc7c5b 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/GCEController.java @@ -74,6 +74,7 @@ * This is a subclass of {@link Controller} that controls load tests on Google Compute Engine. */ public class GCEController extends Controller { + // The standard machine ram is not enough, in some cases. private static final String MACHINE_TYPE = "custom-"; // standard machine prefix private static final String SOURCE_FAMILY = "projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20160930"; // Ubuntu 16.04 LTS @@ -99,6 +100,7 @@ private GCEController(String projectName, Map this.compute = compute; this.projectName = projectName; + // We need to open the zookeeper port first - in case the user provides it. try { createFirewall(); createStorageBucket(); @@ -141,7 +143,6 @@ private GCEController(String projectName, Map } catch (IOException e) { log.debug("Error deleting subscription, assuming it has not yet been created.", e); } - //TODO: fix the AckDeadline - Return it back to 10 instead of 600. try { pubsub.projects().subscriptions().create("projects/" + projectName + "/subscriptions/" + subscription, new Subscription() @@ -165,6 +166,7 @@ private GCEController(String projectName, Map SettableFuture kafkaFuture = SettableFuture.create(); kafkaFutures.add(kafkaFuture); executor.execute(() -> { + // Give the firewall enough time to set everything right. try { Thread.sleep(45000); } catch (InterruptedException e) { } @@ -394,6 +396,7 @@ private void createFirewall() throws IOException { new Firewall.Allowed() .setIPProtocol("tcp") .setPorts(Collections.singletonList("5000")), + // This is the default port for zookeeper access. new Firewall.Allowed() .setIPProtocol("tcp") .setPorts(Collections.singletonList("2181")))); diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java index bb201da0..cd67134c 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/controllers/MessageTracker.java @@ -40,6 +40,7 @@ synchronized void addAllMessageIdentifiers(Iterable identifie identifier -> { receivedMessages.putIfAbsent( identifier.getPublisherClientId(), new HashSet()); + // Needs to create new object, for the test to work correctly. if (!receivedMessages.get( identifier.getPublisherClientId()).add( MessageIdentifier.newBuilder() diff --git a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java index 8caf7183..a6b80100 100644 --- a/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java +++ b/load-test-framework/src/main/java/com/google/pubsub/flic/output/SheetsService.java @@ -71,6 +71,7 @@ public SheetsService(String dataStoreDirectory, Map> types) { types.values().forEach(paramsMap -> { Map countMap = paramsMap.keySet().stream(). @@ -140,6 +141,7 @@ public void sendToSheets(String sheetId, Map>> getValuesList(Map results) { List> cpsValues = new ArrayList<>(results.size()); List> kafkaValues = new ArrayList<>(results.size()); diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java index 7ad9f38c..269d16e2 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/consumer/KafkaConsumer.java @@ -87,7 +87,6 @@ /** * This class, as Kafka's KafkaConsumer, IS NOT THREAD SAFE. - * You must specify Google Cloud project in env variable GOOGLE_CLOUD_PROJECT. * In this implementation, due to differences in Kafka and Pub/Sub behavior, timestamp is treated as an offset. * * This consumer is designed to work with our KafkaProducer implementation. Our KafkaProducer adds attribute "offset" diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java index 0ef107e9..befc1067 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/KafkaProducer.java @@ -67,13 +67,23 @@ import java.util.concurrent.atomic.AtomicBoolean; /** - * A Kafka client that publishes records to Google Cloud Pub/Sub. + * This class, as Kafka's KafkaProducer, is thread safe, the best way to use it is by multithreading on a single producer. + * In this implementation, due to differences in Kafka and Pub/Sub behavior, timestamp set here is used as an offset. + * + * This Producer is designed to work with our KafkaConsumer implementation. + * Value is kept in Pub/Sub message body "data". Key is kept in Pub/Sub attributes map under "key". + * + * @param Key + * @param value */ public class KafkaProducer implements Producer { + // Check if closed method was called previously private final AtomicBoolean closed; + // Identify the version of the producer which have sent the message. private static final long VERSION = 1L; + // The multiplier used with RetrySettings, to mirror that on Kafka's side. private static final double MULTIPLIER = 1.0; private Config producerConfig; @@ -105,6 +115,12 @@ public KafkaProducer(Config configs) { publishers = new ConcurrentHashMap<>(); } + /** + * This method creates a publisher for the provided topic, prior to that it checks if the topic + * is there on the server, if it's not the case it creates it - if auto.create.config is set to true. + * + * @param topic The name of the topic which will be assigned to the publisher. + */ private Publisher createPublisher(String topic) { if (closed.get()) { throw new IllegalStateException("Cannot send after the producer is closed."); @@ -164,7 +180,9 @@ private Publisher createPublisher(String topic) { } /** - * Sends the given record. + * Send the given record. + * + * @param originalRecord The ProducerRecord to be sent. */ public Future send(ProducerRecord originalRecord) { return send(originalRecord, null); @@ -173,13 +191,17 @@ public Future send(ProducerRecord originalRecord) { //TODO: there is an onSendError() in the interceptors, while there are no errors. /** - * Sends the given record and invokes the specified callback. + * Send the given record then invoke the provided callback. + * + * @param originalRecord The ProducerRecord to be sent. + * @param callback The Callback to be invoked. */ public Future send(ProducerRecord originalRecord, Callback callback) { if (closed.get()) { throw new IllegalStateException("Cannot send after the producer is closed."); } + // Send the record to interceptors, if there is any - for pre-processing. ProducerRecord record = producerConfig.getInterceptors() == null ? originalRecord : producerConfig.getInterceptors().onSend(originalRecord); @@ -190,6 +212,7 @@ public Future send(ProducerRecord originalRecord, Callback DateTime dateTime = new DateTime(); + // Serialize the key and value, into raw bytestrings. byte[] valueBytes = null; if (record.value() != null) { try { @@ -229,6 +252,7 @@ public Future send(ProducerRecord originalRecord, Callback final SettableFuture future = SettableFuture.create(); + // issue the callback and notify the interceptors, if there is any. if (callback != null) { if (producerConfig.getAcks()) { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { @@ -258,7 +282,14 @@ public void onSuccess(String result) { return future; } - // Attribute's value's size shouldn't exceed 1024 bytes (256 for key) + /** + * Turn the provided data into a PubsubMessage. + * NOTE: Attribute's value's size shouldn't exceed 1024 bytes (256 for key) + * + * @param dateTime Timestamp when the message was created for sending. + * @param key The bytestring of key. + * @param value the bytestring of value. + */ private PubsubMessage createMessage(Long dateTime, byte[] key, byte[] value) { Map attributes = new HashMap<>(); @@ -278,6 +309,9 @@ private PubsubMessage createMessage(Long dateTime, byte[] key, byte[] value) { .setData(ByteString.copyFrom(value)).putAllAttributes(attributes).build(); } + /** + * Handles the callback thing. + */ private void callbackOnCompletion(Callback cb, RecordMetadata m, Exception e) { if (producerConfig.getInterceptors() != null) { producerConfig.getInterceptors().onAcknowledgement(m, e); @@ -285,13 +319,16 @@ private void callbackOnCompletion(Callback cb, RecordMetadata m, Exception e) { cb.onCompletion(m, e); } + /** + * Wrap provided data into a RecordMetadata object. + */ private RecordMetadata getRecordMetadata(String topic, long offset, int keySize, int valueSize) { return new RecordMetadata(new TopicPartition(topic, 0), offset, 0L, System.currentTimeMillis(), 0L, keySize, valueSize); } /** - * Flushes records that have accumulated. + * Flushes all batches waiting to be sent, it's doing so by shutting down the publishers. */ public void flush() { Map tempPublishers = publishers; @@ -306,24 +343,31 @@ public void flush() { } } + /** + * Not supported on our side - dummied. + */ public List partitionsFor(String topic) { Node[] dummy = {new Node(0, "", 0)}; return Arrays.asList(new PartitionInfo(topic, 0, dummy[0], dummy, dummy)); } + /** + * Not supported on our side - dummied. + */ public Map metrics() { return new HashMap<>(); } /** - * Closes the producer. + * Closes the producer - shutting down all the publishers and setting closed to true. */ public void close() { close(0, null); } /** - * Closes the producer with the given timeout. + * Closes the producer - shutting down all the publishers and setting closed to true. + * The timeout is useless in our implementation. */ public void close(long timeout, TimeUnit unit) { if (timeout < 0) { diff --git a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java index 9e73b120..28c9209e 100644 --- a/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java +++ b/pubsub-mapped-api/src/main/java/com/google/pubsub/clients/producer/PubSubProducerConfig.java @@ -13,13 +13,16 @@ */ public class PubSubProducerConfig extends AbstractConfig { + /** project */ public static final String PROJECT_CONFIG = "project"; private static final String PROJECT_DOC = "GCP project that we will connect to."; + /** elements.count */ public static final String ELEMENTS_COUNT_CONFIG = "elements.count"; private static final String ELEMENTS_COUNT_DOC = "This configuration controls the default count of" + " elements in a batch."; + /** auto.create.topics.enable */ public static final String AUTO_CREATE_TOPICS_CONFIG = "auto.create.topics.enable"; private static final String AUTO_CREATE_TOPICS_DOC = "When true topics are automatically created" + " if they don't exist."; diff --git a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java index ca20d837..9d2549b6 100644 --- a/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java +++ b/pubsub-mapped-api/src/main/java/org/apache/kafka/clients/producer/ProducerConfigCreator.java @@ -21,6 +21,8 @@ /** * This is an adapter to control the ProducerConfig class since it's constructors are package-private. + * It provides a way to instantiate ProducerConfig objects. Also, it propagates the configuration + * with options required by ProducerConfig class that are irrelevant for PubSub. */ public class ProducerConfigCreator {