labels = new ArrayList<>();
+ while (keyIterator.hasNext() && valueIterator.hasNext()) {
+ labels.add(
+ String.format(
+ "%s: \"%s\"", keyIterator.next().getKey(), valueIterator.next().getValue()
+ )
+ );
+ }
+ return String.format(
+ "{namePrefix: \"%s\", labels: [%s], metricRegistry: %s}",
+ getNamePrefix(),
+ String.join(", ", labels),
+ getMetricRegistry()
+ );
+ }
+
/** Creates a new GcpMetricsOptions.Builder. */
public static Builder newBuilder() {
return new Builder();
@@ -269,6 +452,18 @@ public int getUnresponsiveDetectionDroppedCount() {
return unresponsiveDetectionDroppedCount;
}
+ @Override
+ public String toString() {
+ return String.format(
+ "{notReadyFallbackEnabled: %s, unresponsiveDetectionEnabled: %s, " +
+ "unresponsiveDetectionMs: %d, unresponsiveDetectionDroppedCount: %d}",
+ isNotReadyFallbackEnabled(),
+ isUnresponsiveDetectionEnabled(),
+ getUnresponsiveDetectionMs(),
+ getUnresponsiveDetectionDroppedCount()
+ );
+ }
+
public static class Builder {
private boolean notReadyFallbackEnabled = false;
private boolean unresponsiveDetectionEnabled = false;
diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java
new file mode 100644
index 00000000..9d9de7e7
--- /dev/null
+++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java
@@ -0,0 +1,427 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
+import com.google.cloud.grpc.multiendpoint.MultiEndpoint;
+import com.google.cloud.grpc.proto.ApiConfig;
+import com.google.common.base.Preconditions;
+import io.grpc.CallOptions;
+import io.grpc.ClientCall;
+import io.grpc.ClientCall.Listener;
+import io.grpc.ConnectivityState;
+import io.grpc.Grpc;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.opencensus.metrics.LabelKey;
+import io.opencensus.metrics.LabelValue;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * The purpose of GcpMultiEndpointChannel is twofold:
+ *
+ *
+ * - Fallback to an alternative endpoint (host:port) of a gRPC service when the original
+ * endpoint is completely unavailable.
+ *
- Be able to route an RPC call to a specific group of endpoints.
+ *
+ *
+ * A group of endpoints is called a {@link MultiEndpoint} and is essentially a list of endpoints
+ * where priority is defined by the position in the list with the first endpoint having top
+ * priority. A MultiEndpoint tracks endpoints' availability. When a MultiEndpoint is picked for an
+ * RPC call, it picks the top priority endpoint that is currently available. More information on
+ * the {@link MultiEndpoint} class.
+ *
+ *
GcpMultiEndpointChannel can have one or more MultiEndpoint identified by its name -- arbitrary
+ * string provided in the {@link GcpMultiEndpointOptions} when configuring MultiEndpoints. This name
+ * can be used to route an RPC call to this MultiEndpoint by setting the {@link #ME_KEY} key value
+ * of the RPC {@link CallOptions}.
+ *
+ *
GcpMultiEndpointChannel receives a list of GcpMultiEndpointOptions for initial configuration.
+ * An updated configuration can be provided at any time later using
+ * {@link GcpMultiEndpointChannel#setMultiEndpoints(List)}. The first item in the
+ * GcpMultiEndpointOptions list defines the default MultiEndpoint that will be used when no
+ * MultiEndpoint name is provided with an RPC call.
+ *
+ *
Example:
+ *
+ *
Let's assume we have a service with read and write operations and the following backends:
+ *
+ * - service.example.com -- the main set of backends supporting all operations
+ * - service-fallback.example.com -- read-write replica supporting all operations
+ * - ro-service.example.com -- read-only replica supporting only read operations
+ *
+ *
+ * Example configuration:
+ *
+ * -
+ * MultiEndpoint named "default" with endpoints:
+ *
+ * - service.example.com:443
+ * - service-fallback.example.com:443
+ *
+ *
+ * -
+ * MultiEndpoint named "read" with endpoints:
+ *
+ * - ro-service.example.com:443
+ * - service-fallback.example.com:443
+ * - service.example.com:443
+ *
+ *
+ *
+ *
+ * With the configuration above GcpMultiEndpointChannel will use the "default" MultiEndpoint by
+ * default. It means that RPC calls by default will use the main endpoint and if it is not available
+ * then the read-write replica.
+ *
+ *
To offload some read calls to the read-only replica we can specify "read" MultiEndpoint in
+ * the CallOptions. Then these calls will use the read-only replica endpoint and if it is not
+ * available then the read-write replica and if it is also not available then the main endpoint.
+ *
+ *
GcpMultiEndpointChannel creates a {@link GcpManagedChannel} channel pool for every unique
+ * endpoint. For the example above three channel pools will be created.
+ */
+public class GcpMultiEndpointChannel extends ManagedChannel {
+
+ public static final CallOptions.Key ME_KEY = CallOptions.Key.create("MultiEndpoint");
+ private final LabelKey endpointKey =
+ LabelKey.create("endpoint", "Endpoint address.");
+ private final Map multiEndpoints = new ConcurrentHashMap<>();
+ private MultiEndpoint defaultMultiEndpoint;
+ private final ApiConfig apiConfig;
+ private final GcpManagedChannelOptions gcpManagedChannelOptions;
+
+ private final Map pools = new ConcurrentHashMap<>();
+
+ /**
+ * Constructor for {@link GcpMultiEndpointChannel}.
+ *
+ * @param meOptions list of MultiEndpoint configurations.
+ * @param apiConfig the ApiConfig object for configuring GcpManagedChannel.
+ * @param gcpManagedChannelOptions the options for GcpManagedChannel.
+ */
+ public GcpMultiEndpointChannel(
+ List meOptions,
+ ApiConfig apiConfig,
+ GcpManagedChannelOptions gcpManagedChannelOptions) {
+ this.apiConfig = apiConfig;
+ this.gcpManagedChannelOptions = gcpManagedChannelOptions;
+ setMultiEndpoints(meOptions);
+ }
+
+ private class EndpointStateMonitor implements Runnable {
+
+ private final ManagedChannel channel;
+ private final String endpoint;
+
+ private EndpointStateMonitor(ManagedChannel channel, String endpoint) {
+ this.endpoint = endpoint;
+ this.channel = channel;
+ run();
+ }
+
+ @Override
+ public void run() {
+ if (channel == null) {
+ return;
+ }
+ ConnectivityState newState = checkPoolState(channel, endpoint);
+ if (newState != ConnectivityState.SHUTDOWN) {
+ channel.notifyWhenStateChanged(newState, this);
+ }
+ }
+ }
+
+ // Checks and returns channel pool state. Also notifies all MultiEndpoints of the pool state.
+ private ConnectivityState checkPoolState(ManagedChannel channel, String endpoint) {
+ ConnectivityState state = channel.getState(false);
+ // Update endpoint state in all multiendpoints.
+ for (MultiEndpoint me : multiEndpoints.values()) {
+ me.setEndpointAvailable(endpoint, state.equals(ConnectivityState.READY));
+ }
+ return state;
+ }
+
+ private GcpManagedChannelOptions prepareGcpManagedChannelConfig(
+ GcpManagedChannelOptions gcpOptions, String endpoint) {
+ final GcpMetricsOptions.Builder metricsOptions = GcpMetricsOptions.newBuilder(
+ gcpOptions.getMetricsOptions()
+ );
+
+ final List labelKeys = new ArrayList<>(metricsOptions.build().getLabelKeys());
+ final List labelValues = new ArrayList<>(metricsOptions.build().getLabelValues());
+
+ labelKeys.add(endpointKey);
+ labelValues.add(LabelValue.create(endpoint));
+
+ // Make sure the pool will have at least 1 channel always connected. If maximum size > 1 then we
+ // want at least 2 channels or square root of maximum channels whichever is larger.
+ // Do not override if minSize is already specified as > 0.
+ final GcpChannelPoolOptions.Builder poolOptions = GcpChannelPoolOptions.newBuilder(
+ gcpOptions.getChannelPoolOptions()
+ );
+ if (poolOptions.build().getMinSize() < 1) {
+ int minSize = Math.min(2, poolOptions.build().getMaxSize());
+ minSize = Math.max(minSize, ((int) Math.sqrt(poolOptions.build().getMaxSize())));
+ poolOptions.setMinSize(minSize);
+ }
+
+ return GcpManagedChannelOptions.newBuilder(gcpOptions)
+ .withChannelPoolOptions(poolOptions.build())
+ .withMetricsOptions(metricsOptions.withLabels(labelKeys, labelValues).build())
+ .build();
+ }
+
+ /**
+ * Update the list of MultiEndpoint configurations.
+ *
+ * MultiEndpoints are matched with the current ones by name.
+ *
+ * - If a current MultiEndpoint is missing in the updated list, the MultiEndpoint will be
+ * removed.
+ *
- A new MultiEndpoint will be created for every new name in the list.
+ *
- For an existing MultiEndpoint only its endpoints will be updated (no recovery timeout
+ * change).
+ *
+ *
+ * Endpoints are matched by the endpoint address (usually in the form of address:port).
+ *
+ * - If an existing endpoint is not used by any MultiEndpoint in the updated list, then the
+ * channel poll for this endpoint will be shutdown.
+ *
- A channel pool will be created for every new endpoint.
+ *
- For an existing endpoint nothing will change (the channel pool will not be re-created, thus
+ * no channel credentials change, nor channel configurator change).
+ *
+ */
+ public void setMultiEndpoints(List meOptions) {
+ Preconditions.checkNotNull(meOptions);
+ Preconditions.checkArgument(!meOptions.isEmpty(), "MultiEndpoints list is empty");
+ Set currentMultiEndpoints = new HashSet<>();
+ Set currentEndpoints = new HashSet<>();
+
+ // Must have all multiendpoints before initializing the pools so that all multiendpoints
+ // can get status update of every pool.
+ meOptions.forEach(options -> {
+ currentMultiEndpoints.add(options.getName());
+ // Create or update MultiEndpoint
+ if (multiEndpoints.containsKey(options.getName())) {
+ multiEndpoints.get(options.getName()).setEndpoints(options.getEndpoints());
+ } else {
+ multiEndpoints.put(options.getName(),
+ (new MultiEndpoint.Builder(options.getEndpoints()))
+ .withRecoveryTimeout(options.getRecoveryTimeout())
+ .build());
+ }
+ });
+
+ // TODO: Support the same endpoint in different MultiEndpoint to use different channel
+ // credentials.
+ // TODO: Support different endpoints in the same MultiEndpoint to use different channel
+ // credentials.
+ meOptions.forEach(options -> {
+ // Create missing pools
+ options.getEndpoints().forEach(endpoint -> {
+ currentEndpoints.add(endpoint);
+ pools.computeIfAbsent(endpoint, e -> {
+ ManagedChannelBuilder managedChannelBuilder;
+ if (options.getChannelCredentials() != null) {
+ managedChannelBuilder = Grpc.newChannelBuilder(e, options.getChannelCredentials());
+ } else {
+ String serviceAddress;
+ int port;
+ int colon = e.lastIndexOf(':');
+ if (colon < 0) {
+ serviceAddress = e;
+ // Assume https by default.
+ port = 443;
+ } else {
+ serviceAddress = e.substring(0, colon);
+ port = Integer.parseInt(e.substring(colon + 1));
+ }
+ managedChannelBuilder = ManagedChannelBuilder.forAddress(serviceAddress, port);
+ }
+ if (options.getChannelConfigurator() != null) {
+ managedChannelBuilder = options.getChannelConfigurator().apply(managedChannelBuilder);
+ }
+
+ GcpManagedChannel channel = new GcpManagedChannel(
+ managedChannelBuilder,
+ apiConfig,
+ // Add endpoint to metric labels.
+ prepareGcpManagedChannelConfig(gcpManagedChannelOptions, e));
+ // Start monitoring the pool state.
+ new EndpointStateMonitor(channel, e);
+ return channel;
+ });
+ // Communicate current state to MultiEndpoints.
+ checkPoolState(pools.get(endpoint), endpoint);
+ });
+ });
+ defaultMultiEndpoint = multiEndpoints.get(meOptions.get(0).getName());
+
+ // Remove obsolete multiendpoints.
+ multiEndpoints.keySet().removeIf(name -> !currentMultiEndpoints.contains(name));
+
+ // Shutdown and remove the pools not present in options.
+ for (String endpoint : pools.keySet()) {
+ if (!currentEndpoints.contains(endpoint)) {
+ pools.get(endpoint).shutdown();
+ pools.remove(endpoint);
+ }
+ }
+ }
+
+ /**
+ * Initiates an orderly shutdown in which preexisting calls continue but new calls are immediately
+ * cancelled.
+ *
+ * @return this
+ * @since 1.0.0
+ */
+ @Override
+ public ManagedChannel shutdown() {
+ pools.values().forEach(GcpManagedChannel::shutdown);
+ return this;
+ }
+
+ /**
+ * Returns whether the channel is shutdown. Shutdown channels immediately cancel any new calls,
+ * but may still have some calls being processed.
+ *
+ * @see #shutdown()
+ * @see #isTerminated()
+ * @since 1.0.0
+ */
+ @Override
+ public boolean isShutdown() {
+ return pools.values().stream().allMatch(GcpManagedChannel::isShutdown);
+ }
+
+ /**
+ * Returns whether the channel is terminated. Terminated channels have no running calls and
+ * relevant resources released (like TCP connections).
+ *
+ * @see #isShutdown()
+ * @since 1.0.0
+ */
+ @Override
+ public boolean isTerminated() {
+ return pools.values().stream().allMatch(GcpManagedChannel::isTerminated);
+ }
+
+ /**
+ * Initiates a forceful shutdown in which preexisting and new calls are cancelled. Although
+ * forceful, the shutdown process is still not instantaneous; {@link #isTerminated()} will likely
+ * return {@code false} immediately after this method returns.
+ *
+ * @return this
+ * @since 1.0.0
+ */
+ @Override
+ public ManagedChannel shutdownNow() {
+ pools.values().forEach(GcpManagedChannel::shutdownNow);
+ return this;
+ }
+
+ /**
+ * Waits for the channel to become terminated, giving up if the timeout is reached.
+ *
+ * @return whether the channel is terminated, as would be done by {@link #isTerminated()}.
+ * @since 1.0.0
+ */
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ long endTimeNanos = System.nanoTime() + unit.toNanos(timeout);
+ for (GcpManagedChannel gcpManagedChannel : pools.values()) {
+ if (gcpManagedChannel.isTerminated()) {
+ continue;
+ }
+ long awaitTimeNanos = endTimeNanos - System.nanoTime();
+ if (awaitTimeNanos <= 0) {
+ break;
+ }
+ gcpManagedChannel.awaitTermination(awaitTimeNanos, NANOSECONDS);
+ }
+ return isTerminated();
+ }
+
+ /**
+ * Check the value of {@link #ME_KEY} key in the {@link CallOptions} and if found use
+ * the MultiEndpoint with the same name for this call.
+ *
+ * Create a {@link ClientCall} to the remote operation specified by the given {@link
+ * MethodDescriptor}. The returned {@link ClientCall} does not trigger any remote behavior until
+ * {@link ClientCall#start(Listener, Metadata)} is invoked.
+ *
+ * @param methodDescriptor describes the name and parameter types of the operation to call.
+ * @param callOptions runtime options to be applied to this call.
+ * @return a {@link ClientCall} bound to the specified method.
+ * @since 1.0.0
+ */
+ @Override
+ public ClientCall newCall(
+ MethodDescriptor methodDescriptor, CallOptions callOptions) {
+ final String multiEndpointKey = callOptions.getOption(ME_KEY);
+ MultiEndpoint me = defaultMultiEndpoint;
+ if (multiEndpointKey != null) {
+ me = multiEndpoints.getOrDefault(multiEndpointKey, defaultMultiEndpoint);
+ }
+ return pools.get(me.getCurrentId()).newCall(methodDescriptor, callOptions);
+ }
+
+ /**
+ * The authority of the current endpoint of the default MultiEndpoint. Typically, this is in the
+ * format {@code host:port}.
+ *
+ * To get the authority of the current endpoint of another MultiEndpoint use {@link
+ * #authorityFor(String)} method.
+ *
+ * This may return different values over time because MultiEndpoint may switch between endpoints.
+ *
+ * @since 1.0.0
+ */
+ @Override
+ public String authority() {
+ return pools.get(defaultMultiEndpoint.getCurrentId()).authority();
+ }
+
+ /**
+ * The authority of the current endpoint of the specified MultiEndpoint. Typically, this is in the
+ * format {@code host:port}.
+ *
+ * This may return different values over time because MultiEndpoint may switch between endpoints.
+ */
+ public String authorityFor(String multiEndpointName) {
+ MultiEndpoint multiEndpoint = multiEndpoints.get(multiEndpointName);
+ if (multiEndpoint == null) {
+ return null;
+ }
+ return pools.get(multiEndpoint.getCurrentId()).authority();
+ }
+}
diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java
new file mode 100644
index 00000000..c1c30bac
--- /dev/null
+++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import com.google.api.core.ApiFunction;
+import com.google.cloud.grpc.multiendpoint.MultiEndpoint;
+import com.google.common.base.Preconditions;
+import io.grpc.ChannelCredentials;
+import io.grpc.ManagedChannelBuilder;
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * {@link MultiEndpoint} configuration for the {@link GcpMultiEndpointChannel}.
+ */
+public class GcpMultiEndpointOptions {
+
+ private final String name;
+ private final List endpoints;
+ private final ApiFunction, ManagedChannelBuilder> channelConfigurator;
+ private final ChannelCredentials channelCredentials;
+ private final Duration recoveryTimeout;
+
+ public static String DEFAULT_NAME = "default";
+
+ public GcpMultiEndpointOptions(Builder builder) {
+ this.name = builder.name;
+ this.endpoints = builder.endpoints;
+ this.channelConfigurator = builder.channelConfigurator;
+ this.channelCredentials = builder.channelCredentials;
+ this.recoveryTimeout = builder.recoveryTimeout;
+ }
+
+ /**
+ * Creates a new GcpMultiEndpointOptions.Builder.
+ *
+ * @param endpoints list of endpoints for the MultiEndpoint.
+ */
+ public static Builder newBuilder(List endpoints) {
+ return new Builder(endpoints);
+ }
+
+ /**
+ * Creates a new GcpMultiEndpointOptions.Builder from GcpMultiEndpointOptions.
+ */
+ public static Builder newBuilder(GcpMultiEndpointOptions options) {
+ return new Builder(options);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List getEndpoints() {
+ return endpoints;
+ }
+
+ public ApiFunction, ManagedChannelBuilder> getChannelConfigurator() {
+ return channelConfigurator;
+ }
+
+ public ChannelCredentials getChannelCredentials() {
+ return channelCredentials;
+ }
+
+ public Duration getRecoveryTimeout() {
+ return recoveryTimeout;
+ }
+
+ public static class Builder {
+
+ private String name = GcpMultiEndpointOptions.DEFAULT_NAME;
+ private List endpoints;
+ private ApiFunction, ManagedChannelBuilder> channelConfigurator;
+ private ChannelCredentials channelCredentials;
+ private Duration recoveryTimeout = Duration.ZERO;
+
+ public Builder(List endpoints) {
+ setEndpoints(endpoints);
+ }
+
+ public Builder(GcpMultiEndpointOptions options) {
+ this.name = options.getName();
+ this.endpoints = options.getEndpoints();
+ this.channelConfigurator = options.getChannelConfigurator();
+ this.channelCredentials = options.getChannelCredentials();
+ this.recoveryTimeout = options.getRecoveryTimeout();
+ }
+
+ public GcpMultiEndpointOptions build() {
+ return new GcpMultiEndpointOptions(this);
+ }
+
+ private void setEndpoints(List endpoints) {
+ Preconditions.checkNotNull(endpoints);
+ Preconditions.checkArgument(
+ !endpoints.isEmpty(), "At least one endpoint must be specified.");
+ Preconditions.checkArgument(
+ endpoints.stream().noneMatch(s -> s.trim().isEmpty()), "No empty endpoints allowed.");
+ this.endpoints = endpoints;
+ }
+
+ /**
+ * Sets the name of the MultiEndpoint.
+ *
+ * @param name MultiEndpoint name.
+ */
+ public GcpMultiEndpointOptions.Builder withName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ /**
+ * Sets the endpoints of the MultiEndpoint.
+ *
+ * @param endpoints List of endpoints in the form of host:port in descending priority order.
+ */
+ public GcpMultiEndpointOptions.Builder withEndpoints(List endpoints) {
+ this.setEndpoints(endpoints);
+ return this;
+ }
+
+ /**
+ * Sets the channel configurator for the MultiEndpoint channel pool.
+ *
+ * @param channelConfigurator function to perform on the ManagedChannelBuilder in the channel
+ * pool.
+ */
+ public GcpMultiEndpointOptions.Builder withChannelConfigurator(
+ ApiFunction, ManagedChannelBuilder> channelConfigurator) {
+ this.channelConfigurator = channelConfigurator;
+ return this;
+ }
+
+ /**
+ * Sets the channel credentials to use in the MultiEndpoint channel pool.
+ *
+ * @param channelCredentials channel credentials.
+ */
+ public GcpMultiEndpointOptions.Builder withChannelCredentials(
+ ChannelCredentials channelCredentials) {
+ this.channelCredentials = channelCredentials;
+ return this;
+ }
+
+ /**
+ * Sets the recovery timeout for the MultiEndpoint. See more info in the {@link MultiEndpoint}.
+ *
+ * @param recoveryTimeout recovery timeout.
+ */
+ public GcpMultiEndpointOptions.Builder withRecoveryTimeout(Duration recoveryTimeout) {
+ this.recoveryTimeout = recoveryTimeout;
+ return this;
+ }
+ }
+}
diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java
new file mode 100644
index 00000000..fc6d08eb
--- /dev/null
+++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc.multiendpoint;
+
+import com.google.errorprone.annotations.CheckReturnValue;
+import java.util.concurrent.ScheduledFuture;
+
+/**
+ * Endpoint holds an endpoint's state, priority and a future of upcoming state change.
+ */
+@CheckReturnValue
+final class Endpoint {
+
+ /**
+ * Holds a state of an endpoint.
+ */
+ public enum EndpointState {
+ UNAVAILABLE,
+ AVAILABLE,
+ RECOVERING,
+ }
+
+ private final String id;
+ private EndpointState state;
+ private int priority;
+ private ScheduledFuture changeStateFuture;
+
+ public Endpoint(String id, EndpointState state, int priority) {
+ this.id = id;
+ this.priority = priority;
+ this.state = state;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public EndpointState getState() {
+ return state;
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public void setState(EndpointState state) {
+ this.state = state;
+ }
+
+ public void setPriority(int priority) {
+ this.priority = priority;
+ }
+
+ public synchronized void setChangeStateFuture(ScheduledFuture future) {
+ resetStateChangeFuture();
+ changeStateFuture = future;
+ }
+
+ public synchronized void resetStateChangeFuture() {
+ if (changeStateFuture != null) {
+ changeStateFuture.cancel(true);
+ }
+ }
+}
diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java
new file mode 100644
index 00000000..18b9abfd
--- /dev/null
+++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc.multiendpoint;
+
+import static java.util.Comparator.comparingInt;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import com.google.cloud.grpc.multiendpoint.Endpoint.EndpointState;
+import com.google.common.base.Preconditions;
+import com.google.errorprone.annotations.CheckReturnValue;
+import com.google.errorprone.annotations.concurrent.GuardedBy;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+/**
+ * MultiEndpoint holds a list of endpoints, tracks their availability and defines the current
+ * endpoint. An endpoint has a priority defined by its position in the list (first item has top
+ * priority). MultiEndpoint returns top priority endpoint that is available as current. If no
+ * endpoint is available, MultiEndpoint returns the top priority endpoint.
+ *
+ * Sometimes switching between endpoints can be costly, and it is worth waiting for some time
+ * after current endpoint becomes unavailable. For this case, use {@link
+ * Builder#withRecoveryTimeout} to set the recovery timeout. MultiEndpoint will keep the current
+ * endpoint for up to recovery timeout after it became unavailable to give it some time to recover.
+ *
+ *
The list of endpoints can be changed at any time with {@link #setEndpoints} method.
+ * MultiEndpoint will preserve endpoints' state and update their priority according to their new
+ * positions.
+ *
+ *
The initial state of endpoint is "unavailable" or "recovering" if using recovery timeout.
+ */
+@CheckReturnValue
+public final class MultiEndpoint {
+ @GuardedBy("this")
+ private final Map endpointsMap = new HashMap<>();
+
+ @GuardedBy("this")
+ private String currentId;
+
+ private final Duration recoveryTimeout;
+
+ private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
+
+ private MultiEndpoint(Builder builder) {
+ this.recoveryTimeout = builder.recoveryTimeout;
+ this.setEndpoints(builder.endpoints);
+ }
+
+ /** Builder for MultiEndpoint. */
+ public static final class Builder {
+ private final List endpoints;
+ private Duration recoveryTimeout = Duration.ZERO;
+
+ public Builder(List endpoints) {
+ Preconditions.checkNotNull(endpoints);
+ Preconditions.checkArgument(!endpoints.isEmpty(), "Endpoints list must not be empty.");
+ this.endpoints = endpoints;
+ }
+
+ /**
+ * MultiEndpoint will keep the current endpoint for up to recovery timeout after it became
+ * unavailable to give it some time to recover.
+ */
+ public Builder withRecoveryTimeout(Duration timeout) {
+ Preconditions.checkNotNull(timeout);
+ this.recoveryTimeout = timeout;
+ return this;
+ }
+
+ public MultiEndpoint build() {
+ return new MultiEndpoint(this);
+ }
+ }
+
+ /**
+ * Returns current endpoint id.
+ *
+ * Note that the read is not synchronized and in case of a race condition there is a chance of
+ * getting an outdated current id.
+ */
+ @SuppressWarnings("GuardedBy")
+ public String getCurrentId() {
+ return currentId;
+ }
+
+ private synchronized void setEndpointStateInternal(String endpointId, EndpointState state) {
+ Endpoint endpoint = endpointsMap.get(endpointId);
+ if (endpoint != null) {
+ endpoint.setState(state);
+ maybeUpdateCurrentEndpoint();
+ }
+ }
+
+ private boolean isRecoveryEnabled() {
+ return !recoveryTimeout.isNegative() && !recoveryTimeout.isZero();
+ }
+
+ /** Inform MultiEndpoint when an endpoint becomes available or unavailable. */
+ public synchronized void setEndpointAvailable(String endpointId, boolean available) {
+ setEndpointState(endpointId, available ? EndpointState.AVAILABLE : EndpointState.UNAVAILABLE);
+ }
+
+ private synchronized void setEndpointState(String endpointId, EndpointState state) {
+ Preconditions.checkNotNull(state);
+ Endpoint endpoint = endpointsMap.get(endpointId);
+ if (endpoint == null) {
+ return;
+ }
+ // If we allow some recovery time.
+ if (EndpointState.UNAVAILABLE.equals(state) && isRecoveryEnabled()) {
+ endpoint.setState(EndpointState.RECOVERING);
+ ScheduledFuture future =
+ executor.schedule(
+ () -> setEndpointStateInternal(endpointId, EndpointState.UNAVAILABLE),
+ recoveryTimeout.toMillis(),
+ MILLISECONDS);
+ endpoint.setChangeStateFuture(future);
+ return;
+ }
+ endpoint.resetStateChangeFuture();
+ endpoint.setState(state);
+ maybeUpdateCurrentEndpoint();
+ }
+
+ /**
+ * Provide an updated list of endpoints to MultiEndpoint.
+ *
+ *
MultiEndpoint will preserve current endpoints' state and update their priority according to
+ * their new positions.
+ */
+ public synchronized void setEndpoints(List endpoints) {
+ Preconditions.checkNotNull(endpoints);
+ Preconditions.checkArgument(!endpoints.isEmpty(), "Endpoints list must not be empty.");
+
+ // Remove obsolete endpoints.
+ endpointsMap.keySet().retainAll(endpoints);
+
+ // Add new endpoints and update priority.
+ int priority = 0;
+ for (String endpointId : endpoints) {
+ Endpoint existingEndpoint = endpointsMap.get(endpointId);
+ if (existingEndpoint != null) {
+ existingEndpoint.setPriority(priority++);
+ continue;
+ }
+ EndpointState newState =
+ isRecoveryEnabled() ? EndpointState.RECOVERING : EndpointState.UNAVAILABLE;
+ Endpoint newEndpoint = new Endpoint(endpointId, newState, priority++);
+ if (isRecoveryEnabled()) {
+ ScheduledFuture future =
+ executor.schedule(
+ () -> setEndpointStateInternal(endpointId, EndpointState.UNAVAILABLE),
+ recoveryTimeout.toMillis(),
+ MILLISECONDS);
+ newEndpoint.setChangeStateFuture(future);
+ }
+ endpointsMap.put(endpointId, newEndpoint);
+ }
+
+ maybeUpdateCurrentEndpoint();
+ }
+
+ // Updates currentId to the top-priority available endpoint unless the current endpoint is
+ // recovering.
+ private synchronized void maybeUpdateCurrentEndpoint() {
+ Optional topEndpoint =
+ endpointsMap.values().stream()
+ .filter((c) -> c.getState().equals(EndpointState.AVAILABLE))
+ .min(comparingInt(Endpoint::getPriority));
+
+ Endpoint current = endpointsMap.get(currentId);
+ if (current != null && current.getState().equals(EndpointState.RECOVERING)) {
+ // Keep recovering endpoint as current unless a higher priority endpoint became available.
+ if (!topEndpoint.isPresent() || topEndpoint.get().getPriority() >= current.getPriority()) {
+ return;
+ }
+ }
+
+ if (!topEndpoint.isPresent() && current == null) {
+ topEndpoint = endpointsMap.values().stream().min(comparingInt(Endpoint::getPriority));
+ }
+
+ topEndpoint.ifPresent(endpoint -> currentId = endpoint.getId());
+ }
+}
diff --git a/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto b/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto
index 1301dd99..81987f19 100644
--- a/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto
+++ b/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto
@@ -21,14 +21,19 @@ option java_outer_classname = "GcpExtensionProto";
option java_package = "com.google.cloud.grpc.proto";
message ApiConfig {
+ // Deprecated. Use GcpManagedChannelOptions.GcpChannelPoolOptions class.
// The channel pool configurations.
- ChannelPoolConfig channel_pool = 2;
+ ChannelPoolConfig channel_pool = 2 [deprecated = true];
// The method configurations.
repeated MethodConfig method = 1001;
}
+
+// Deprecated. Use GcpManagedChannelOptions.GcpChannelPoolOptions class.
message ChannelPoolConfig {
+ option deprecated = true;
+
// The max number of channels in the pool.
uint32 max_size = 1;
// The idle timeout (seconds) of channels without bound affinity sessions.
diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
index 3a39d86a..c8ddb9bd 100644
--- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
+++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
@@ -16,11 +16,13 @@
package com.google.cloud.grpc;
+import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
import io.opencensus.metrics.LabelKey;
@@ -34,7 +36,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/** Unit tests for GcpManagedChannel. */
+/** Unit tests for GcpManagedChannelOptionsTest. */
@RunWith(JUnit4.class)
public final class GcpManagedChannelOptionsTest {
private static final String namePrefix = "name-prefix";
@@ -168,4 +170,25 @@ public void testOptionsReBuild() {
assertEquals(unresponsiveMs, resOpts.getUnresponsiveDetectionMs());
assertEquals(unresponsiveDroppedCount, resOpts.getUnresponsiveDetectionDroppedCount());
}
+
+ @Test
+ public void testPoolOptions() {
+ final GcpManagedChannelOptions opts = GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(5)
+ .setMinSize(2)
+ .setConcurrentStreamsLowWatermark(10)
+ .setUseRoundRobinOnBind(true)
+ .build()
+ )
+ .build();
+
+ GcpChannelPoolOptions channelPoolOptions = opts.getChannelPoolOptions();
+ assertThat(channelPoolOptions).isNotNull();
+ assertThat(channelPoolOptions.getMaxSize()).isEqualTo(5);
+ assertThat(channelPoolOptions.getMinSize()).isEqualTo(2);
+ assertThat(channelPoolOptions.getConcurrentStreamsLowWatermark()).isEqualTo(10);
+ assertThat(channelPoolOptions.isUseRoundRobinOnBind()).isTrue();
+ }
}
diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
index 50c3b3c6..9af0e88e 100644
--- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
+++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertNotNull;
import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
import com.google.cloud.grpc.MetricRegistryTestUtils.FakeMetricRegistry;
@@ -49,9 +50,18 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -70,6 +80,38 @@ public final class GcpManagedChannelTest {
private static final int MAX_CHANNEL = 10;
private static final int MAX_STREAM = 100;
+ private static final Logger testLogger = Logger.getLogger(GcpManagedChannel.class.getName());
+
+ private final List logRecords = new LinkedList<>();
+
+ private String lastLogMessage() {
+ return lastLogMessage(1);
+ }
+
+ private String lastLogMessage(int nthFromLast) {
+ return logRecords.get(logRecords.size() - nthFromLast).getMessage();
+ }
+
+ private Level lastLogLevel() {
+ return lastLogLevel(1);
+ }
+
+ private Level lastLogLevel(int nthFromLast) {
+ return logRecords.get(logRecords.size() - nthFromLast).getLevel();
+ }
+
+ private final Handler testLogHandler = new Handler() {
+ @Override
+ public synchronized void publish(LogRecord record) {
+ logRecords.add(record);
+ }
+
+ @Override
+ public void flush() {}
+
+ @Override
+ public void close() throws SecurityException {}
+ };
private GcpManagedChannel gcpChannel;
private ManagedChannelBuilder builder;
@@ -82,6 +124,7 @@ private void resetGcpChannel() {
@Before
public void setUpChannel() {
+ testLogger.addHandler(testLogHandler);
builder = ManagedChannelBuilder.forAddress(TARGET, 443);
gcpChannel = (GcpManagedChannel) GcpManagedChannelBuilder.forDelegateBuilder(builder).build();
}
@@ -89,6 +132,9 @@ public void setUpChannel() {
@After
public void shutdown() {
gcpChannel.shutdownNow();
+ testLogger.removeHandler(testLogHandler);
+ testLogger.setLevel(Level.INFO);
+ logRecords.clear();
}
@Test
@@ -130,17 +176,117 @@ public void testLoadApiConfigString() throws Exception {
assertEquals(3, gcpChannel.methodToAffinity.size());
}
+ @Test
+ public void testUsesPoolOptions() {
+ resetGcpChannel();
+ GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(5)
+ .setMinSize(2)
+ .setConcurrentStreamsLowWatermark(50)
+ .build();
+ GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions)
+ .build();
+ gcpChannel =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(builder)
+ .withOptions(options)
+ .build();
+ assertEquals(2, gcpChannel.channelRefs.size());
+ assertEquals(5, gcpChannel.getMaxSize());
+ assertEquals(2, gcpChannel.getMinSize());
+ assertEquals(50, gcpChannel.getStreamsLowWatermark());
+ }
+
+ @Test
+ public void testPoolOptionsOverrideApiConfig() {
+ resetGcpChannel();
+ final URL resource = GcpManagedChannelTest.class.getClassLoader().getResource(API_FILE);
+ assertNotNull(resource);
+ File configFile = new File(resource.getFile());
+ GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(5)
+ .setConcurrentStreamsLowWatermark(50)
+ .build();
+ GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions)
+ .build();
+ gcpChannel =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(builder)
+ .withApiConfigJsonFile(configFile)
+ .withOptions(options)
+ .build();
+ assertEquals(0, gcpChannel.channelRefs.size());
+ assertEquals(5, gcpChannel.getMaxSize());
+ assertEquals(50, gcpChannel.getStreamsLowWatermark());
+ assertEquals(3, gcpChannel.methodToAffinity.size());
+ }
+
@Test
public void testGetChannelRefInitialization() {
+ // Watch debug messages.
+ testLogger.setLevel(Level.FINER);
+
+ final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
+ final String poolIndex = String.format("pool-%d", currentIndex);
+
+ // Initial log messages count.
+ int logCount = logRecords.size();
+
// Should not have a managedchannel by default.
assertEquals(0, gcpChannel.channelRefs.size());
// But once requested it's there.
assertEquals(0, gcpChannel.getChannelRef(null).getAffinityCount());
+
+ assertThat(logRecords.size()).isEqualTo(logCount + 2);
+ assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Channel 0 created.");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINER);
+ assertThat(logRecords.get(logRecords.size() - 2).getMessage()).isEqualTo(
+ poolIndex + ": Channel 0 state change detected: null -> IDLE");
+ assertThat(logRecords.get(logRecords.size() - 2).getLevel()).isEqualTo(Level.FINER);
+
// The state of this channel is idle.
assertEquals(ConnectivityState.IDLE, gcpChannel.getState(false));
assertEquals(1, gcpChannel.channelRefs.size());
}
+ @Test
+ public void testGetChannelRefInitializationWithMinSize() throws InterruptedException {
+ resetGcpChannel();
+ GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(5)
+ .setMinSize(2)
+ .build();
+ GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions)
+ .build();
+ gcpChannel =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(builder)
+ .withOptions(options)
+ .build();
+ // Should have 2 channels since the beginning.
+ assertThat(gcpChannel.channelRefs.size()).isEqualTo(2);
+ TimeUnit.MILLISECONDS.sleep(50);
+ // The connection establishment must have been started on these two channels.
+ assertThat(gcpChannel.getState(false))
+ .isAnyOf(
+ ConnectivityState.CONNECTING,
+ ConnectivityState.READY,
+ ConnectivityState.TRANSIENT_FAILURE);
+ assertThat(gcpChannel.channelRefs.get(0).getChannel().getState(false))
+ .isAnyOf(
+ ConnectivityState.CONNECTING,
+ ConnectivityState.READY,
+ ConnectivityState.TRANSIENT_FAILURE);
+ assertThat(gcpChannel.channelRefs.get(1).getChannel().getState(false))
+ .isAnyOf(
+ ConnectivityState.CONNECTING,
+ ConnectivityState.READY,
+ ConnectivityState.TRANSIENT_FAILURE);
+ }
+
@Test
public void testGetChannelRefPickUpSmallest() {
// All channels have max number of streams
@@ -179,6 +325,9 @@ private void assertFallbacksMetric(
@Test
public void testGetChannelRefWithFallback() {
+ // Watch debug messages.
+ testLogger.setLevel(Level.FINEST);
+
final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry();
final int maxSize = 3;
@@ -205,6 +354,9 @@ public void testGetChannelRefWithFallback() {
.build())
.build();
+ final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
+ final String poolIndex = String.format("pool-%d", currentIndex);
+
// Creates the first channel with 0 id.
assertEquals(0, pool.getNumberOfChannels());
ChannelRef chRef = pool.getChannelRef(null);
@@ -219,20 +371,35 @@ public void testGetChannelRefWithFallback() {
// Let's simulate the non-ready state for the 0 channel.
pool.processChannelStateChange(0, ConnectivityState.CONNECTING);
+ int logCount = logRecords.size();
// Now request for a channel should return a newly created channel because our current channel
- // is not ready and we haven't reached the pool's max size.
+ // is not ready, and we haven't reached the pool's max size.
chRef = pool.getChannelRef(null);
assertEquals(1, chRef.getId());
assertEquals(2, pool.getNumberOfChannels());
+ // This was a fallback from non-ready channel 0 to the newly created channel 1.
+ assertThat(logRecords.size()).isEqualTo(logCount + 3);
+ logRecords.forEach(logRecord -> System.out.println(logRecord.getMessage()));
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Fallback to newly created channel 1");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 1, 0);
// Adding one active stream to channel 1.
pool.channelRefs.get(1).activeStreamsCountIncr();
+ logCount = logRecords.size();
// Having 0 active streams on channel 0 and 1 active streams on channel one with the default
// settings would return channel 0 for the next channel request. But having fallback enabled and
// channel 0 not ready it should return channel 1 instead.
chRef = pool.getChannelRef(null);
assertEquals(1, chRef.getId());
assertEquals(2, pool.getNumberOfChannels());
+ // This was the second fallback from non-ready channel 0 to the channel 1.
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Picking fallback channel: 0 -> 1");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 2, 0);
// Now let's have channel 0 still as not ready but bring channel 1 streams to low watermark.
for (int i = 0; i < lowWatermark - 1; i++) {
@@ -259,6 +426,8 @@ public void testGetChannelRefWithFallback() {
chRef = pool.getChannelRef(null);
assertEquals(2, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
+ // This was the third fallback from non-ready channel 0 to the channel 2.
+ assertFallbacksMetric(fakeRegistry, 3, 0);
// Let's bring channel 1 to max streams and mark channel 2 as not ready.
for (int i = 0; i < MAX_STREAM - lowWatermark; i++) {
@@ -269,54 +438,88 @@ public void testGetChannelRefWithFallback() {
// Now we have two non-ready channels and one overloaded.
// Even when fallback enabled there is no good candidate at this time, the next channel request
- // should return a channel with lowest streams count regardless of its readiness state.
+ // should return a channel with the lowest streams count regardless of its readiness state.
// In our case it is channel 0.
+ logCount = logRecords.size();
chRef = pool.getChannelRef(null);
assertEquals(0, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
-
- // So far the fallback logic sometimes provided different channels than a pool with disabled
- // fallback would provide. But for metrics we consider a fallback only if we have an affinity
- // key that was mapped to some channel and after that channel went to a non-ready state we
- // temporarily used another channel as a fallback.
- // Because of that, metric values for successful and failed fallbacks should be still zero.
- assertFallbacksMetric(fakeRegistry, 0, 0);
+ // This will also count as a failed fallback because we couldn't find a ready and non-overloaded
+ // channel.
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Failed to find fallback for channel 0");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 3, 1);
// Let's have an affinity key and bind it to channel 0.
final String key = "ABC";
pool.bind(pool.channelRefs.get(0), Collections.singletonList(key));
+ logCount = logRecords.size();
// Channel 0 is not ready currently and the fallback enabled should look for a fallback but we
// still don't have a good channel because channel 1 is not ready and channel 2 is overloaded.
// The getChannelRef should return the original channel 0 and report a failed fallback.
chRef = pool.getChannelRef(key);
assertEquals(0, chRef.getId());
- assertFallbacksMetric(fakeRegistry, 0, 1);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Failed to find fallback for channel 0");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 3, 2);
// Let's return channel 1 to a ready state.
pool.processChannelStateChange(1, ConnectivityState.READY);
+ logCount = logRecords.size();
// Now we have a fallback candidate.
// The getChannelRef should return the channel 1 and report a successful fallback.
chRef = pool.getChannelRef(key);
assertEquals(1, chRef.getId());
- assertFallbacksMetric(fakeRegistry, 1, 1);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Setting fallback channel: 0 -> 1");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 4, 2);
+
+ // Let's briefly bring channel 2 to ready state.
+ pool.processChannelStateChange(2, ConnectivityState.READY);
+ logCount = logRecords.size();
+ // Now we have a better fallback candidate (fewer streams on channel 2). But this time we
+ // already used channel 1 as a fallback, and we should stick to it instead of returning the
+ // original channel.
+ // The getChannelRef should return the channel 1 and report a successful fallback.
+ chRef = pool.getChannelRef(key);
+ assertEquals(1, chRef.getId());
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Using fallback channel: 0 -> 1");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 5, 2);
+ pool.processChannelStateChange(2, ConnectivityState.CONNECTING);
// Let's bring channel 1 back to connecting state.
pool.processChannelStateChange(1, ConnectivityState.CONNECTING);
+ logCount = logRecords.size();
// Now we don't have a good fallback candidate again. But this time we already used channel 1
// as a fallback and we should stick to it instead of returning the original channel.
// The getChannelRef should return the channel 1 and report a failed fallback.
chRef = pool.getChannelRef(key);
assertEquals(1, chRef.getId());
- assertFallbacksMetric(fakeRegistry, 1, 2);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Failed to find fallback for channel 0");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+ assertFallbacksMetric(fakeRegistry, 5, 3);
// Finally, we bring both channel 1 and channel 0 to the ready state and we should get the
// original channel 0 for the key without any fallbacks happening.
pool.processChannelStateChange(1, ConnectivityState.READY);
pool.processChannelStateChange(0, ConnectivityState.READY);
+ logCount = logRecords.size();
chRef = pool.getChannelRef(key);
assertEquals(0, chRef.getId());
- assertFallbacksMetric(fakeRegistry, 1, 2);
+ assertThat(logRecords.size()).isEqualTo(logCount);
+ assertFallbacksMetric(fakeRegistry, 5, 3);
}
@Test
@@ -333,13 +536,30 @@ public void testGetChannelRefMaxSize() {
@Test
public void testBindUnbindKey() {
+ // Watch debug messages.
+ testLogger.setLevel(Level.FINEST);
+
+ final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
+ final String poolIndex = String.format("pool-%d", currentIndex);
+
// Initialize the channel and bind the key, check the affinity count.
ChannelRef cf1 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 5);
- ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 4);
+ ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 2, 0, 4);
gcpChannel.channelRefs.add(cf1);
gcpChannel.channelRefs.add(cf2);
+
gcpChannel.bind(cf1, Collections.singletonList("key1"));
+
+ // Initial log messages count.
+ int logCount = logRecords.size();
+
gcpChannel.bind(cf2, Collections.singletonList("key2"));
+
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Binding 1 key(s) to channel 2: [key2]");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
+
gcpChannel.bind(cf2, Collections.singletonList("key3"));
// Binding the same key to the same channel should not increase affinity count.
gcpChannel.bind(cf1, Collections.singletonList("key1"));
@@ -352,15 +572,25 @@ public void testBindUnbindKey() {
assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount());
assertEquals(3, gcpChannel.affinityKeyToChannelRef.size());
+ logCount = logRecords.size();
+
// Unbind the affinity key.
gcpChannel.unbind(Collections.singletonList("key1"));
assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount());
assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount());
assertEquals(2, gcpChannel.affinityKeyToChannelRef.size());
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Unbinding key key1 from channel 1.");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
gcpChannel.unbind(Collections.singletonList("key1"));
assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount());
assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount());
assertEquals(2, gcpChannel.affinityKeyToChannelRef.size());
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": Unbinding key key1 but it wasn't bound.");
+ assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
gcpChannel.unbind(Collections.singletonList("key2"));
assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount());
assertEquals(0, gcpChannel.channelRefs.get(1).getAffinityCount());
@@ -371,6 +601,27 @@ public void testBindUnbindKey() {
assertEquals(0, gcpChannel.affinityKeyToChannelRef.size());
}
+ @Test
+ public void testUsingKeyWithoutBinding() {
+ // Initialize the channel and bind the key, check the affinity count.
+ ChannelRef cf1 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 5);
+ ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 2, 0, 4);
+ gcpChannel.channelRefs.add(cf1);
+ gcpChannel.channelRefs.add(cf2);
+
+ final String key = "non-binded-key";
+ ChannelRef channelRef = gcpChannel.getChannelRef(key);
+ // Should bind on the fly to the least busy channel, which is 2.
+ assertThat(channelRef.getId()).isEqualTo(2);
+
+ cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
+ cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
+ channelRef = gcpChannel.getChannelRef(key);
+ // Even after channel 1 now has less active streams (3) the channel 2 is still mapped for the
+ // same key.
+ assertThat(channelRef.getId()).isEqualTo(2);
+ }
+
@Test
public void testGetKeysFromRequest() {
String expected = "thisisaname";
@@ -483,6 +734,8 @@ public void testParseEmptyChannelJsonFile() {
@Test
public void testMetrics() {
+ // Watch debug messages.
+ testLogger.setLevel(Level.FINE);
final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry();
final String prefix = "some/prefix/";
final List labelKeys =
@@ -510,12 +763,24 @@ public void testMetrics() {
.build())
.build();
+ final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
+ final String poolIndex = String.format("pool-%d", currentIndex);
+
+ // Logs metrics options.
+ assertThat(logRecords.get(logRecords.size() - 2).getLevel()).isEqualTo(Level.FINE);
+ assertThat(logRecords.get(logRecords.size() - 2).getMessage()).startsWith(
+ poolIndex + ": Metrics options: {namePrefix: \"some/prefix/\", labels: " +
+ "[key_a: \"val_a\", key_b: \"val_b\"],"
+ );
+
+ assertThat(lastLogLevel()).isEqualTo(Level.INFO);
+ assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Metrics enabled.");
+
List expectedLabelKeys = new ArrayList<>(labelKeys);
expectedLabelKeys.add(
LabelKey.create(GcpMetricsConstants.POOL_INDEX_LABEL, GcpMetricsConstants.POOL_INDEX_DESC));
List expectedLabelValues = new ArrayList<>(labelValues);
- int currentIndex = GcpManagedChannel.channelPoolIndex.get();
- expectedLabelValues.add(LabelValue.create(String.format("pool-%d", currentIndex)));
+ expectedLabelValues.add(LabelValue.create(poolIndex));
try {
// Let's fill five channels with some fake streams.
@@ -530,12 +795,19 @@ public void testMetrics() {
MetricsRecord record = fakeRegistry.pollRecord();
assertThat(record.getMetrics().size()).isEqualTo(25);
+ // Initial log messages count.
+ int logCount = logRecords.size();
+
List> numChannels =
record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_CHANNELS);
assertThat(numChannels.size()).isEqualTo(1);
assertThat(numChannels.get(0).value()).isEqualTo(5L);
assertThat(numChannels.get(0).keys()).isEqualTo(expectedLabelKeys);
assertThat(numChannels.get(0).values()).isEqualTo(expectedLabelValues);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogLevel()).isEqualTo(Level.FINE);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_CHANNELS + " = 5");
List> maxAllowedChannels =
record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS);
@@ -543,6 +815,10 @@ public void testMetrics() {
assertThat(maxAllowedChannels.get(0).value()).isEqualTo(MAX_CHANNEL);
assertThat(maxAllowedChannels.get(0).keys()).isEqualTo(expectedLabelKeys);
assertThat(maxAllowedChannels.get(0).values()).isEqualTo(expectedLabelValues);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogLevel()).isEqualTo(Level.FINE);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS + " = 10");
List> minActiveStreams =
record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS);
@@ -550,6 +826,10 @@ public void testMetrics() {
assertThat(minActiveStreams.get(0).value()).isEqualTo(0L);
assertThat(minActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys);
assertThat(minActiveStreams.get(0).values()).isEqualTo(expectedLabelValues);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogLevel()).isEqualTo(Level.FINE);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS + " = 0");
List> maxActiveStreams =
record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS);
@@ -557,21 +837,247 @@ public void testMetrics() {
assertThat(maxActiveStreams.get(0).value()).isEqualTo(7L);
assertThat(maxActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys);
assertThat(maxActiveStreams.get(0).values()).isEqualTo(expectedLabelValues);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogLevel()).isEqualTo(Level.FINE);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS + " = 7");
List> totalActiveStreams =
record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS);
assertThat(totalActiveStreams.size()).isEqualTo(1);
- assertThat(totalActiveStreams.get(0).value())
- .isEqualTo(Arrays.stream(streams).asLongStream().sum());
+ long totalStreamsExpected = Arrays.stream(streams).asLongStream().sum();
+ assertThat(totalActiveStreams.get(0).value()).isEqualTo(totalStreamsExpected);
assertThat(totalActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys);
assertThat(totalActiveStreams.get(0).values()).isEqualTo(expectedLabelValues);
+ assertThat(logRecords.size()).isEqualTo(++logCount);
+ assertThat(lastLogLevel()).isEqualTo(Level.FINE);
+ assertThat(lastLogMessage()).isEqualTo(
+ poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS + " = " +
+ totalStreamsExpected);
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testLogMetrics() throws InterruptedException {
+ // Watch debug messages.
+ testLogger.setLevel(Level.FINE);
+
+ final GcpManagedChannel pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(builder)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(5)
+ .setConcurrentStreamsLowWatermark(3)
+ .build())
+ .withMetricsOptions(
+ GcpMetricsOptions.newBuilder()
+ .withNamePrefix("prefix")
+ .build())
+ .withResiliencyOptions(
+ GcpResiliencyOptions.newBuilder()
+ .setNotReadyFallback(true)
+ .withUnresponsiveConnectionDetection(100, 2)
+ .build())
+ .build())
+ .build();
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ try {
+ final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
+ final String poolIndex = String.format("pool-%d", currentIndex);
+
+ int[] streams = new int[]{3, 2, 5, 7, 1};
+ int[] keyCount = new int[]{2, 3, 1, 1, 4};
+ int[] okCalls = new int[]{2, 2, 8, 2, 3};
+ int[] errCalls = new int[]{1, 1, 2, 2, 1};
+ List channels = new ArrayList<>();
+ for (int i = 0; i < streams.length; i++) {
+ FakeManagedChannel channel = new FakeManagedChannel(executorService);
+ channels.add(channel);
+ ChannelRef ref = pool.new ChannelRef(channel, i);
+ pool.channelRefs.add(ref);
+
+ // Simulate channel connecting.
+ channel.setState(ConnectivityState.CONNECTING);
+ TimeUnit.MILLISECONDS.sleep(10);
+
+ // For the last one...
+ if (i == streams.length - 1) {
+ // This will be a couple of successful fallbacks.
+ pool.getChannelRef(null);
+ pool.getChannelRef(null);
+ // Bring down all other channels.
+ for (int j = 0; j < i; j++) {
+ channels.get(j).setState(ConnectivityState.CONNECTING);
+ }
+ TimeUnit.MILLISECONDS.sleep(100);
+ // And this will be a failed fallback (no ready channels).
+ pool.getChannelRef(null);
+
+ // Simulate unresponsive connection.
+ long startNanos = System.nanoTime();
+ final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED);
+ ref.activeStreamsCountIncr();
+ ref.activeStreamsCountDecr(startNanos, deStatus, false);
+ ref.activeStreamsCountIncr();
+ ref.activeStreamsCountDecr(startNanos, deStatus, false);
+
+ // Simulate unresponsive connection with more dropped calls.
+ startNanos = System.nanoTime();
+ ref.activeStreamsCountIncr();
+ ref.activeStreamsCountDecr(startNanos, deStatus, false);
+ ref.activeStreamsCountIncr();
+ ref.activeStreamsCountDecr(startNanos, deStatus, false);
+ TimeUnit.MILLISECONDS.sleep(110);
+ ref.activeStreamsCountIncr();
+ ref.activeStreamsCountDecr(startNanos, deStatus, false);
+ }
+
+ channel.setState(ConnectivityState.READY);
+
+ for (int j = 0; j < streams[i]; j++) {
+ ref.activeStreamsCountIncr();
+ }
+ // Bind affinity keys.
+ final List keys = new ArrayList<>();
+ for (int j = 0; j < keyCount[i]; j++) {
+ keys.add("key-" + i + "-" + j);
+ }
+ pool.bind(ref, keys);
+ // Simulate successful calls.
+ for (int j = 0; j < okCalls[i]; j++) {
+ ref.activeStreamsCountDecr(0, Status.OK, false);
+ ref.activeStreamsCountIncr();
+ }
+ // Simulate failed calls.
+ for (int j = 0; j < errCalls[i]; j++) {
+ ref.activeStreamsCountDecr(0, Status.UNAVAILABLE, false);
+ ref.activeStreamsCountIncr();
+ }
+
+ }
+
+ logRecords.clear();
+
+ pool.logMetrics();
+
+ List