From 158b06d2e23b045aa8aaf69e303920bf4c55f0b7 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Nov 2021 12:33:21 -0500 Subject: [PATCH 01/17] merge RefreshingManagedChannel & SafeShutdownManagedChannel into ChannelPool. The eventual goal is to allow channel pool to safely add remove channels. The code has been refactored roughly as: - SafeShutdownManagedChannel is now ChannelPool.Entry and ReleasingClientCall - RefreshingManagedChannel has been merged into ChannelPool as a pair of functions scheduleNextRefresh & refresh --- .../com/google/api/gax/grpc/ChannelPool.java | 294 +++++++++++++++--- .../google/api/gax/grpc/ChannelPoolTest.java | 8 +- 2 files changed, 258 insertions(+), 44 deletions(-) diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java index d6b85275b..010b722e6 100644 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java +++ b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java @@ -32,32 +32,57 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.grpc.CallOptions; +import io.grpc.Channel; import io.grpc.ClientCall; +import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; +import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; import io.grpc.ManagedChannel; +import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.grpc.Status; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; +import org.threeten.bp.Duration; /** * A {@link ManagedChannel} that will send requests round robin via a set of channels. * + *

In addition to spreading requests over a set of child connections, the pool will also actively + * manage the lifecycle of the channels. Currently lifecycle management is limited to pre-emptively + * replacing channels every hour. In the future it will dynamically size the pool based on number of + * outstanding requests. + * *

Package-private for internal use. */ class ChannelPool extends ManagedChannel { + private static final Logger LOG = Logger.getLogger(ChannelPool.class.getName()); + // size greater than 1 to allow multiple channel to refresh at the same time // size not too large so refreshing channels doesn't use too many threads private static final int CHANNEL_REFRESH_EXECUTOR_SIZE = 2; - private final ImmutableList channels; + private static final Duration REFRESH_PERIOD = Duration.ofMinutes(50); + private static final double JITTER_PERCENTAGE = 0.15; + + // A copy on write list of child channels. + private final AtomicReference> entries = new AtomicReference<>(); private final AtomicInteger indexTicker = new AtomicInteger(); private final String authority; // if set, ChannelPool will manage the life cycle of channelRefreshExecutorService - @Nullable private ScheduledExecutorService channelRefreshExecutorService; + @Nullable private final ScheduledExecutorService channelRefreshExecutorService; + private final ChannelFactory channelFactory; + + private volatile ScheduledFuture nextScheduledRefresh = null; /** * Factory method to create a non-refreshing channel pool @@ -66,12 +91,8 @@ class ChannelPool extends ManagedChannel { * @param channelFactory method to create the channels * @return ChannelPool of non refreshing channels */ - static ChannelPool create(int poolSize, final ChannelFactory channelFactory) throws IOException { - List channels = new ArrayList<>(); - for (int i = 0; i < poolSize; i++) { - channels.add(channelFactory.createSingleChannel()); - } - return new ChannelPool(channels, null); + static ChannelPool create(int poolSize, ChannelFactory channelFactory) throws IOException { + return new ChannelPool(channelFactory, poolSize, null); } /** @@ -88,14 +109,10 @@ static ChannelPool create(int poolSize, final ChannelFactory channelFactory) thr @VisibleForTesting static ChannelPool createRefreshing( int poolSize, - final ChannelFactory channelFactory, + ChannelFactory channelFactory, ScheduledExecutorService channelRefreshExecutorService) throws IOException { - List channels = new ArrayList<>(); - for (int i = 0; i < poolSize; i++) { - channels.add(new RefreshingManagedChannel(channelFactory, channelRefreshExecutorService)); - } - return new ChannelPool(channels, channelRefreshExecutorService); + return new ChannelPool(channelFactory, poolSize, channelRefreshExecutorService); } /** @@ -114,15 +131,30 @@ static ChannelPool createRefreshing(int poolSize, final ChannelFactory channelFa /** * Initializes the channel pool. Assumes that all channels have the same authority. * - * @param channels a List of channels to pool. + * @param channelFactory method to create the channels + * @param poolSize number of channels in the pool * @param channelRefreshExecutorService periodically refreshes the channels */ private ChannelPool( - List channels, - @Nullable ScheduledExecutorService channelRefreshExecutorService) { - this.channels = ImmutableList.copyOf(channels); - authority = channels.get(0).authority(); + ChannelFactory channelFactory, + int poolSize, + @Nullable ScheduledExecutorService channelRefreshExecutorService) + throws IOException { + this.channelFactory = channelFactory; + + ImmutableList.Builder initialListBuilder = ImmutableList.builder(); + + for (int i = 0; i < poolSize; i++) { + initialListBuilder.add(new Entry(channelFactory.createSingleChannel())); + } + + entries.set(initialListBuilder.build()); + authority = entries.get().get(0).channel.authority(); this.channelRefreshExecutorService = channelRefreshExecutorService; + + if (channelRefreshExecutorService != null) { + nextScheduledRefresh = scheduleNextRefresh(); + } } /** {@inheritDoc} */ @@ -140,17 +172,26 @@ public String authority() { @Override public ClientCall newCall( MethodDescriptor methodDescriptor, CallOptions callOptions) { - return getNextChannel().newCall(methodDescriptor, callOptions); + return getChannel(indexTicker.getAndIncrement()).newCall(methodDescriptor, callOptions); + } + + Channel getChannel(int affinity) { + return new AffinityChannel(affinity); } /** {@inheritDoc} */ @Override public ManagedChannel shutdown() { - for (ManagedChannel channelWrapper : channels) { - channelWrapper.shutdown(); + List localEntries = entries.get(); + for (Entry entry : localEntries) { + entry.channel.shutdown(); + } + if (nextScheduledRefresh != null) { + nextScheduledRefresh.cancel(true); } if (channelRefreshExecutorService != null) { - channelRefreshExecutorService.shutdown(); + // shutdownNow will cancel scheduled tasks + channelRefreshExecutorService.shutdownNow(); } return this; } @@ -158,8 +199,9 @@ public ManagedChannel shutdown() { /** {@inheritDoc} */ @Override public boolean isShutdown() { - for (ManagedChannel channel : channels) { - if (!channel.isShutdown()) { + List localEntries = entries.get(); + for (Entry entry : localEntries) { + if (!entry.channel.isShutdown()) { return false; } } @@ -172,8 +214,9 @@ public boolean isShutdown() { /** {@inheritDoc} */ @Override public boolean isTerminated() { - for (ManagedChannel channel : channels) { - if (!channel.isTerminated()) { + List localEntries = entries.get(); + for (Entry entry : localEntries) { + if (!entry.channel.isTerminated()) { return false; } } @@ -186,8 +229,12 @@ public boolean isTerminated() { /** {@inheritDoc} */ @Override public ManagedChannel shutdownNow() { - for (ManagedChannel channel : channels) { - channel.shutdownNow(); + List localEntries = entries.get(); + for (Entry entry : localEntries) { + entry.channel.shutdownNow(); + } + if (nextScheduledRefresh != null) { + nextScheduledRefresh.cancel(true); } if (channelRefreshExecutorService != null) { channelRefreshExecutorService.shutdownNow(); @@ -199,12 +246,13 @@ public ManagedChannel shutdownNow() { @Override public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { long endTimeNanos = System.nanoTime() + unit.toNanos(timeout); - for (ManagedChannel channel : channels) { + List localEntries = entries.get(); + for (Entry entry : localEntries) { long awaitTimeNanos = endTimeNanos - System.nanoTime(); if (awaitTimeNanos <= 0) { break; } - channel.awaitTermination(awaitTimeNanos, TimeUnit.NANOSECONDS); + entry.channel.awaitTermination(awaitTimeNanos, TimeUnit.NANOSECONDS); } if (channelRefreshExecutorService != null) { long awaitTimeNanos = endTimeNanos - System.nanoTime(); @@ -213,16 +261,61 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE return isTerminated(); } + /** Scheduling loop. */ + private ScheduledFuture scheduleNextRefresh() { + long delayPeriod = REFRESH_PERIOD.toMillis(); + long jitter = (long) ((Math.random() - 0.5) * JITTER_PERCENTAGE * delayPeriod); + long delay = jitter + delayPeriod; + return channelRefreshExecutorService.schedule( + () -> { + try { + refresh(); + } finally { + scheduleNextRefresh(); + } + }, + delay, + TimeUnit.MILLISECONDS); + } + /** - * Performs a simple round robin on the list of {@link ManagedChannel}s in the {@code channels} - * list. + * Replace all of the channels in the channel pool with fresh ones. This is meant to mitigate the + * hourly GFE disconnects by giving clients the ability to prime the channel on reconnect. * - * @return A {@link ManagedChannel} that can be used for a single RPC call. + *

This is done on a best effort basis. If the replacement channel fails to construct, the old + * channel will continue to be used. */ - private ManagedChannel getNextChannel() { - return getChannel(indexTicker.getAndIncrement()); + private void refresh() { + List localEntries = entries.get(); + ArrayList newEntries = new ArrayList<>(localEntries); + ArrayList removedEntries = new ArrayList<>(); + + for (int i = 0; i < localEntries.size(); i++) { + try { + Entry removed = newEntries.set(i, new Entry(channelFactory.createSingleChannel())); + removedEntries.add(removed); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to refresh channel, leaving old channel", e); + } + } + entries.set(ImmutableList.copyOf(newEntries)); + + removedEntries.forEach(Entry::requestShutdown); } + /** + * Get and retain a Channel Entry. The returned will have its rpc count incremented, preventing it + * from getting recycled. + */ + Entry getRetainedEntry(int affinity) { + for (int i = 0; i < 5; i++) { + Entry entry = getEntry(affinity); + if (entry.retain()) { + return entry; + } + } + throw new IllegalStateException("Failed to retain a channel"); + } /** * Returns one of the channels managed by this pool. The pool continues to "own" the channel, and * the caller should not shut it down. @@ -231,13 +324,136 @@ private ManagedChannel getNextChannel() { * reverse is not true: Two calls with different affinities might return the same channel. * However, the implementation should attempt to spread load evenly. */ - ManagedChannel getChannel(int affinity) { - int index = affinity % channels.size(); + private Entry getEntry(int affinity) { + List localEntries = entries.get(); + + int index = affinity % localEntries.size(); index = Math.abs(index); // If index is the most negative int, abs(index) is still negative. if (index < 0) { index = 0; } - return channels.get(index); + + return localEntries.get(index); + } + + /** Bundles a gRPC {@link ManagedChannel} with some usage accounting. */ + private static class Entry { + private final ManagedChannel channel; + private final AtomicInteger outstandingRpcs = new AtomicInteger(0); + + // Flag that the channel should be closed once all of the outstanding RPC complete. + private final AtomicBoolean shutdownRequested = new AtomicBoolean(); + // Flag that the channel has been closed. + private final AtomicBoolean shutdownInitiated = new AtomicBoolean(); + + private Entry(ManagedChannel channel) { + this.channel = channel; + } + + /** + * Try to increment the outstanding RPC count. The method will return false if the channel is + * closing and the caller should pick a different channel. If the method returned true, the + * channel has been successfully retained and it is the responsibility of the caller to release + * it. + */ + private boolean retain() { + // register desire to start RPC + outstandingRpcs.incrementAndGet(); + + // abort if the channel is closing + if (shutdownRequested.get()) { + release(); + return false; + } + return true; + } + + /** + * Notify the channel that the number of outstanding RPCs has decreased. If shutdown has been + * previously requested, this method will shutdown the channel if its the last outstanding RPC. + */ + private void release() { + int newCount = outstandingRpcs.decrementAndGet(); + if (newCount < 0) { + LOG.log(Level.SEVERE, "Reference count is negative!: " + newCount); + } + + if (newCount == 0 && shutdownRequested.get()) { + shutdown(); + } + } + + /** + * Request a shutdown. The actual shutdown will be delayed until there are no more outstanding + * RPCs. + */ + private void requestShutdown() { + shutdownRequested.set(true); + if (outstandingRpcs.get() == 0) { + shutdown(); + } + } + + /** Ensure that shutdown is only called once. */ + private void shutdown() { + if (shutdownInitiated.compareAndSet(false, true)) { + channel.shutdown(); + } + } + } + + /** Thin wrapper to ensure that new calls are properly reference counted. */ + private class AffinityChannel extends Channel { + private final int affinity; + + public AffinityChannel(int affinity) { + this.affinity = affinity; + } + + @Override + public String authority() { + return authority; + } + + @Override + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + + Entry entry = getRetainedEntry(affinity); + + return new ReleasingClientCall<>(entry.channel.newCall(methodDescriptor, callOptions), entry); + } + } + + /** ClientCall wrapper that makes sure to decrement the outstanding RPC count on completion. */ + static class ReleasingClientCall extends SimpleForwardingClientCall { + final Entry entry; + + public ReleasingClientCall(ClientCall delegate, Entry entry) { + super(delegate); + this.entry = entry; + } + + @Override + public void start(Listener responseListener, Metadata headers) { + try { + super.start( + new SimpleForwardingClientCallListener(responseListener) { + @Override + public void onClose(Status status, Metadata trailers) { + try { + super.onClose(status, trailers); + } finally { + entry.release(); + } + } + }, + headers); + } catch (Exception e) { + // In case start failed, make sure to release + entry.release(); + } + } } } diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java index d6ae65202..875a0e97a 100644 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java +++ b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java @@ -91,15 +91,13 @@ private void verifyTargetChannel( @SuppressWarnings("unchecked") ClientCall expectedClientCall = Mockito.mock(ClientCall.class); - for (ManagedChannel channel : channels) { - Mockito.reset(channel); - } + channels.forEach(Mockito::reset); Mockito.doReturn(expectedClientCall).when(targetChannel).newCall(methodDescriptor, callOptions); ClientCall actualCall = pool.newCall(methodDescriptor, callOptions); - - Truth.assertThat(actualCall).isSameInstanceAs(expectedClientCall); Mockito.verify(targetChannel, Mockito.times(1)).newCall(methodDescriptor, callOptions); + actualCall.start(null, null); + Mockito.verify(expectedClientCall, Mockito.times(1)).start(Mockito.any(), Mockito.any()); for (ManagedChannel otherChannel : channels) { if (otherChannel != targetChannel) { From 7e02e4ab068573ef049b29f8d2833ef1adf30980 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Nov 2021 12:48:09 -0500 Subject: [PATCH 02/17] migrate SafeShutdownManagedChannel tests --- .../com/google/api/gax/grpc/ChannelPool.java | 4 +- .../google/api/gax/grpc/ChannelPoolTest.java | 151 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java index 010b722e6..2c73b1c54 100644 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java +++ b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java @@ -29,6 +29,7 @@ */ package com.google.api.gax.grpc; +import com.google.api.core.InternalApi; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.grpc.CallOptions; @@ -285,7 +286,8 @@ private ScheduledFuture scheduleNextRefresh() { *

This is done on a best effort basis. If the replacement channel fails to construct, the old * channel will continue to be used. */ - private void refresh() { + @InternalApi("Visible for testing") + void refresh() { List localEntries = entries.get(); ArrayList newEntries = new ArrayList<>(localEntries); ArrayList removedEntries = new ArrayList<>(); diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java index 875a0e97a..5ce13f5aa 100644 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java +++ b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java @@ -30,7 +30,9 @@ package com.google.api.gax.grpc; import com.google.api.gax.grpc.testing.FakeChannelFactory; +import com.google.api.gax.grpc.testing.FakeMethodDescriptor; import com.google.api.gax.grpc.testing.FakeServiceGrpc; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.truth.Truth; import com.google.type.Color; @@ -38,7 +40,9 @@ import io.grpc.CallOptions; import io.grpc.ClientCall; import io.grpc.ManagedChannel; +import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.grpc.Status; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -228,4 +232,151 @@ public void channelPrimerIsCalledPeriodically() throws IOException { Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.eq(TimeUnit.MILLISECONDS)); scheduledExecutorService.shutdown(); } + + // ---- + // call should be allowed to complete and the channel should not shutdown + @Test + public void callShouldCompleteAfterCreation() throws IOException { + final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); + ManagedChannel replacementChannel = Mockito.mock(ManagedChannel.class); + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); + ChannelPool pool = ChannelPool.create(1, channelFactory); + + // create a mock call when new call comes to the underlying channel + MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); + MockClientCall spyClientCall = Mockito.spy(mockClientCall); + Mockito.when( + underlyingChannel.newCall( + Mockito.>any(), Mockito.any(CallOptions.class))) + .thenReturn(spyClientCall); + + Answer verifyChannelNotShutdown = + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); + return invocation.callRealMethod(); + } + }; + + // verify that underlying channel is not shutdown when clientCall is still sending message + Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); + + // create a new call on entry + @SuppressWarnings("unchecked") + ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); + ClientCall call = + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + pool.refresh(); + // shutdown is not called because there is still an outstanding call, even if it hasn't started + Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); + + // start clientCall + call.start(listener, new Metadata()); + // send message and end the call + call.sendMessage("message"); + // shutdown is called because the outstanding call has completed + Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); + + // Replacement channel shouldn't be touched + Mockito.verify(replacementChannel, Mockito.never()).shutdown(); + Mockito.verify(replacementChannel, Mockito.never()).newCall(Mockito.any(), Mockito.any()); + } + + // call should be allowed to complete and the channel should not shutdown + @Test + public void callShouldCompleteAfterStarted() throws IOException { + final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); + ManagedChannel replacementChannel = Mockito.mock(ManagedChannel.class); + + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); + ChannelPool pool = ChannelPool.create(1, channelFactory); + + // create a mock call when new call comes to the underlying channel + MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); + MockClientCall spyClientCall = Mockito.spy(mockClientCall); + Mockito.when( + underlyingChannel.newCall( + Mockito.>any(), Mockito.any(CallOptions.class))) + .thenReturn(spyClientCall); + + Answer verifyChannelNotShutdown = + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); + return invocation.callRealMethod(); + } + }; + + // verify that underlying channel is not shutdown when clientCall is still sending message + Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); + + // create a new call on safeShutdownManagedChannel + @SuppressWarnings("unchecked") + ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); + ClientCall call = + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + // start clientCall + call.start(listener, new Metadata()); + pool.refresh(); + + // shutdown is not called because there is still an outstanding call + Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); + // send message and end the call + call.sendMessage("message"); + // shutdown is called because the outstanding call has completed + Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); + } + + // Channel should shutdown after a refresh all the calls have completed + @Test + public void channelShouldShutdown() throws IOException { + final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); + ManagedChannel replacementChannel = Mockito.mock(ManagedChannel.class); + + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); + ChannelPool pool = ChannelPool.create(1, channelFactory); + + // create a mock call when new call comes to the underlying channel + MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); + MockClientCall spyClientCall = Mockito.spy(mockClientCall); + Mockito.when( + underlyingChannel.newCall( + Mockito.>any(), Mockito.any(CallOptions.class))) + .thenReturn(spyClientCall); + + Answer verifyChannelNotShutdown = + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); + return invocation.callRealMethod(); + } + }; + + // verify that underlying channel is not shutdown when clientCall is still sending message + Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); + + // create a new call on safeShutdownManagedChannel + @SuppressWarnings("unchecked") + ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); + ClientCall call = + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + // start clientCall + call.start(listener, new Metadata()); + // send message and end the call + call.sendMessage("message"); + // shutdown is not called because it has not been shutdown yet + Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); + pool.refresh(); + // shutdown is called because the outstanding call has completed + Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); + } } From 50735bdcf39906cb2e88b70bb0d031b5599c3ab1 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Nov 2021 13:04:53 -0500 Subject: [PATCH 03/17] migrate old tests and remove RefreshingManagedChannel and SafeShutdownManagedChannel --- .../gax/grpc/RefreshingManagedChannel.java | 215 ------------------ .../gax/grpc/SafeShutdownManagedChannel.java | 170 -------------- .../google/api/gax/grpc/ChannelPoolTest.java | 47 +++- .../grpc/RefreshingManagedChannelTest.java | 193 ---------------- .../grpc/SafeShutdownManagedChannelTest.java | 176 -------------- 5 files changed, 44 insertions(+), 757 deletions(-) delete mode 100644 gax-grpc/src/main/java/com/google/api/gax/grpc/RefreshingManagedChannel.java delete mode 100644 gax-grpc/src/main/java/com/google/api/gax/grpc/SafeShutdownManagedChannel.java delete mode 100644 gax-grpc/src/test/java/com/google/api/gax/grpc/RefreshingManagedChannelTest.java delete mode 100644 gax-grpc/src/test/java/com/google/api/gax/grpc/SafeShutdownManagedChannelTest.java diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/RefreshingManagedChannel.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/RefreshingManagedChannel.java deleted file mode 100644 index ec8680d4c..000000000 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/RefreshingManagedChannel.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google LLC nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.api.gax.grpc; - -import io.grpc.CallOptions; -import io.grpc.ClientCall; -import io.grpc.ManagedChannel; -import io.grpc.MethodDescriptor; -import java.io.IOException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.threeten.bp.Duration; - -/** - * A {@link ManagedChannel} that will refresh the underlying channel by swapping the underlying - * channel with a new one periodically - * - *

Package-private for internal use. - * - *

A note on the synchronization logic. refreshChannel is called periodically which updates - * delegate and nextScheduledRefresh. lock is needed to provide atomic access and update of delegate - * and nextScheduledRefresh. One example is newCall needs to be atomic to avoid context switching to - * refreshChannel that shuts down delegate before newCall is completed. - */ -class RefreshingManagedChannel extends ManagedChannel { - private static final Logger LOG = Logger.getLogger(RefreshingManagedChannel.class.getName()); - // refresh every 50 minutes with 15% jitter for a range of 42.5min to 57.5min - private static final Duration refreshPeriod = Duration.ofMinutes(50); - private static final double jitterPercentage = 0.15; - private volatile SafeShutdownManagedChannel delegate; - private volatile ScheduledFuture nextScheduledRefresh; - // Read: method calls on delegate and nextScheduledRefresh - // Write: updating references of delegate and nextScheduledRefresh - private final ReadWriteLock lock; - private final ChannelFactory channelFactory; - private final ScheduledExecutorService scheduledExecutorService; - - RefreshingManagedChannel( - ChannelFactory channelFactory, ScheduledExecutorService scheduledExecutorService) - throws IOException { - this.delegate = new SafeShutdownManagedChannel(channelFactory.createSingleChannel()); - this.channelFactory = channelFactory; - this.scheduledExecutorService = scheduledExecutorService; - this.lock = new ReentrantReadWriteLock(); - this.nextScheduledRefresh = scheduleNextRefresh(); - } - - /** - * Refresh the existing channel by swapping the current channel with a new channel and schedule - * the next refresh - * - *

refreshChannel can only be called by scheduledExecutorService and not any other methods in - * this class. This is important so no threads will try to acquire the write lock while holding - * the read lock. - */ - private void refreshChannel() { - SafeShutdownManagedChannel newChannel; - try { - newChannel = new SafeShutdownManagedChannel(channelFactory.createSingleChannel()); - } catch (IOException ioException) { - LOG.log( - Level.WARNING, - "Failed to create a new channel when refreshing channel. This has no effect on the " - + "existing channels. The existing channel will continue to be used", - ioException); - return; - } - - SafeShutdownManagedChannel oldChannel = delegate; - lock.writeLock().lock(); - try { - // This thread can be interrupted by invoking cancel on nextScheduledRefresh - // Interrupt happens when this thread is blocked on acquiring the write lock because shutdown - // was called and that thread holds the read lock. - // When shutdown completes and releases the read lock and this thread acquires the write lock. - // This thread should not continue because the channel has shutdown. This check ensures that - // this thread terminates without swapping the channel and do not schedule the next refresh. - if (Thread.currentThread().isInterrupted()) { - newChannel.shutdownNow(); - return; - } - delegate = newChannel; - nextScheduledRefresh = scheduleNextRefresh(); - } finally { - lock.writeLock().unlock(); - } - oldChannel.shutdownSafely(); - } - - /** Schedule the next instance of refreshing this channel */ - private ScheduledFuture scheduleNextRefresh() { - long delayPeriod = refreshPeriod.toMillis(); - long jitter = (long) ((Math.random() - 0.5) * jitterPercentage * delayPeriod); - long delay = jitter + delayPeriod; - return scheduledExecutorService.schedule( - new Runnable() { - @Override - public void run() { - refreshChannel(); - } - }, - delay, - TimeUnit.MILLISECONDS); - } - - /** {@inheritDoc} */ - @Override - public ClientCall newCall( - MethodDescriptor methodDescriptor, CallOptions callOptions) { - lock.readLock().lock(); - try { - return delegate.newCall(methodDescriptor, callOptions); - } finally { - lock.readLock().unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public String authority() { - // no lock here because authority is constant across all channels - return delegate.authority(); - } - - /** {@inheritDoc} */ - @Override - public ManagedChannel shutdown() { - lock.readLock().lock(); - try { - nextScheduledRefresh.cancel(true); - delegate.shutdown(); - return this; - } finally { - lock.readLock().unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public ManagedChannel shutdownNow() { - lock.readLock().lock(); - try { - nextScheduledRefresh.cancel(true); - delegate.shutdownNow(); - return this; - } finally { - lock.readLock().unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public boolean isShutdown() { - lock.readLock().lock(); - try { - return delegate.isShutdown(); - } finally { - lock.readLock().unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public boolean isTerminated() { - lock.readLock().lock(); - try { - return delegate.isTerminated(); - } finally { - lock.readLock().unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - lock.readLock().lock(); - try { - return delegate.awaitTermination(timeout, unit); - } finally { - lock.readLock().unlock(); - } - } -} diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/SafeShutdownManagedChannel.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/SafeShutdownManagedChannel.java deleted file mode 100644 index 0ae71d2e3..000000000 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/SafeShutdownManagedChannel.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google LLC nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.api.gax.grpc; - -import com.google.common.base.Preconditions; -import io.grpc.CallOptions; -import io.grpc.ClientCall; -import io.grpc.ClientCall.Listener; -import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; -import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; -import io.grpc.ManagedChannel; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.Status; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * A {@link ManagedChannel} that will complete all calls started on the underlying channel before - * shutting down. - * - *

This class is not thread-safe. Caller must synchronize in order to ensure no new calls if safe - * shutdown has started. - * - *

Package-private for internal use. - */ -class SafeShutdownManagedChannel extends ManagedChannel { - private final ManagedChannel delegate; - private final AtomicInteger outstandingCalls = new AtomicInteger(0); - private volatile boolean isShutdownSafely = false; - - SafeShutdownManagedChannel(ManagedChannel managedChannel) { - this.delegate = managedChannel; - } - - /** - * Safely shutdown channel by checking that there are no more outstanding calls. If there are - * outstanding calls, the last call will invoke this method again when it complete - * - *

Caller should take care to synchronize with newCall so no new calls are started after - * shutdownSafely is called - */ - void shutdownSafely() { - isShutdownSafely = true; - if (outstandingCalls.get() == 0) { - delegate.shutdown(); - } - } - - /** {@inheritDoc} */ - @Override - public ManagedChannel shutdown() { - delegate.shutdown(); - return this; - } - - /** {@inheritDoc} */ - @Override - public boolean isShutdown() { - return delegate.isShutdown(); - } - - /** {@inheritDoc} */ - @Override - public ManagedChannel shutdownNow() { - delegate.shutdownNow(); - return this; - } - - /** {@inheritDoc} */ - @Override - public boolean isTerminated() { - return delegate.isTerminated(); - } - - /** {@inheritDoc} */ - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return delegate.awaitTermination(timeout, unit); - } - - /** - * Decrement outstanding call counter and shutdown if there are no more outstanding calls and - * {@link SafeShutdownManagedChannel#shutdownSafely()} has been invoked - */ - private void onClientCallClose() { - if (outstandingCalls.decrementAndGet() == 0 && isShutdownSafely) { - shutdownSafely(); - } - } - - /** Listener that's responsible for decrementing outstandingCalls when the call closes */ - private class DecrementOutstandingCalls extends SimpleForwardingClientCallListener { - DecrementOutstandingCalls(Listener delegate) { - super(delegate); - } - - @Override - public void onClose(Status status, Metadata trailers) { - // decrement in finally block in case onClose throws an exception - try { - super.onClose(status, trailers); - } finally { - onClientCallClose(); - } - } - } - - /** To wrap around delegate to hook in {@link DecrementOutstandingCalls} */ - private class ClientCallProxy extends SimpleForwardingClientCall { - ClientCallProxy(ClientCall delegate) { - super(delegate); - } - - @Override - public void start(Listener responseListener, Metadata headers) { - super.start(new DecrementOutstandingCalls<>(responseListener), headers); - } - } - - /** - * Caller must take care to synchronize newCall and shutdownSafely in order to avoid race - * conditions of starting new calls after shutdownSafely is called - * - * @see io.grpc.ManagedChannel#newCall(MethodDescriptor, CallOptions) - */ - @Override - public ClientCall newCall( - MethodDescriptor methodDescriptor, CallOptions callOptions) { - Preconditions.checkState(!isShutdownSafely); - // increment after client call in case newCall throws an exception - ClientCall clientCall = - new ClientCallProxy<>(delegate.newCall(methodDescriptor, callOptions)); - outstandingCalls.incrementAndGet(); - return clientCall; - } - - /** {@inheritDoc} */ - @Override - public String authority() { - return delegate.authority(); - } -} diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java index 5ce13f5aa..53a1447b2 100644 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java +++ b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java @@ -184,9 +184,9 @@ public void channelPrimerShouldCallPoolConstruction() throws IOException { @Test public void channelPrimerIsCalledPeriodically() throws IOException { ChannelPrimer mockChannelPrimer = Mockito.mock(ChannelPrimer.class); - ManagedChannel channel1 = Mockito.mock(RefreshingManagedChannel.class); - ManagedChannel channel2 = Mockito.mock(RefreshingManagedChannel.class); - ManagedChannel channel3 = Mockito.mock(RefreshingManagedChannel.class); + ManagedChannel channel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel channel2 = Mockito.mock(ManagedChannel.class); + ManagedChannel channel3 = Mockito.mock(ManagedChannel.class); List channelRefreshers = new ArrayList<>(); @@ -379,4 +379,45 @@ public Object answer(InvocationOnMock invocation) throws Throwable { // shutdown is called because the outstanding call has completed Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); } + + @Test + public void channelRefreshShouldSwapChannels() throws IOException { + ManagedChannel underlyingChannel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel underlyingChannel2 = Mockito.mock(ManagedChannel.class); + + // mock executor service to capture the runnable scheduled so we can invoke it when we want to + ScheduledExecutorService scheduledExecutorService = + Mockito.mock(ScheduledExecutorService.class); + final List channelRefreshers = new ArrayList<>(); + Answer extractChannelRefresher = + new Answer() { + public Object answer(InvocationOnMock invocation) { + channelRefreshers.add(invocation.getArgument(0)); + return null; + } + }; + + Mockito.doAnswer(extractChannelRefresher) + .when(scheduledExecutorService) + .schedule( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.eq(TimeUnit.MILLISECONDS)); + + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel1, underlyingChannel2)); + ChannelPool pool = ChannelPool.createRefreshing(1, channelFactory, scheduledExecutorService); + Mockito.reset(underlyingChannel1); + + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + Mockito.verify(underlyingChannel1, Mockito.only()) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // swap channel + pool.refresh(); + + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + Mockito.verify(underlyingChannel2, Mockito.only()) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + } } diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/RefreshingManagedChannelTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/RefreshingManagedChannelTest.java deleted file mode 100644 index b56070bf3..000000000 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/RefreshingManagedChannelTest.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google LLC nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.api.gax.grpc; - -import com.google.api.gax.grpc.testing.FakeChannelFactory; -import com.google.api.gax.grpc.testing.FakeMethodDescriptor; -import io.grpc.CallOptions; -import io.grpc.ClientCall; -import io.grpc.ManagedChannel; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.Status; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; - -@RunWith(JUnit4.class) -public class RefreshingManagedChannelTest { - @Test - public void channelRefreshShouldSwapChannels() throws IOException { - ManagedChannel underlyingChannel1 = Mockito.mock(ManagedChannel.class); - ManagedChannel underlyingChannel2 = Mockito.mock(ManagedChannel.class); - - // mock executor service to capture the runnable scheduled so we can invoke it when we want to - ScheduledExecutorService scheduledExecutorService = - Mockito.mock(ScheduledExecutorService.class); - List channelRefreshers = new ArrayList<>(); - Answer extractChannelRefresher = - invocation -> { - channelRefreshers.add((Runnable) invocation.getArgument(0)); - return null; - }; - - Mockito.doAnswer(extractChannelRefresher) - .when(scheduledExecutorService) - .schedule( - Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.eq(TimeUnit.MILLISECONDS)); - - FakeChannelFactory channelFactory = - new FakeChannelFactory(Arrays.asList(underlyingChannel1, underlyingChannel2)); - - ManagedChannel refreshingManagedChannel = - new RefreshingManagedChannel(channelFactory, scheduledExecutorService); - - refreshingManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - - Mockito.verify(underlyingChannel1, Mockito.only()) - .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); - - // swap channel - channelRefreshers.get(0).run(); - - refreshingManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - - Mockito.verify(underlyingChannel2, Mockito.only()) - .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); - } - - @Test - public void randomizeTest() throws IOException, InterruptedException, ExecutionException { - int channelCount = 10; - ManagedChannel[] underlyingChannels = new ManagedChannel[channelCount]; - Random r = new Random(); - for (int i = 0; i < channelCount; i++) { - ManagedChannel mockManagedChannel = Mockito.mock(ManagedChannel.class); - underlyingChannels[i] = mockManagedChannel; - - Answer waitAndSendMessage = - invocation -> { - // add a little time to sleep so calls don't always complete right away - TimeUnit.MICROSECONDS.sleep(r.nextInt(1000)); - // when sending message on the call, the channel cannot be shutdown - Mockito.verify(mockManagedChannel, Mockito.never()).shutdown(); - return invocation.callRealMethod(); - }; - - Answer createNewCall = - invocation -> { - // create a new client call for every new call to the underlying channel - MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); - MockClientCall spyClientCall = Mockito.spy(mockClientCall); - - // spy into clientCall to verify that the channel is not shutdown - Mockito.doAnswer(waitAndSendMessage) - .when(spyClientCall) - .sendMessage(Mockito.anyString()); - - return spyClientCall; - }; - - // return a new mocked client call when requesting new call on the channel - Mockito.doAnswer(createNewCall) - .when(underlyingChannels[i]) - .newCall( - Mockito.>any(), Mockito.any(CallOptions.class)); - } - - // mock executor service to capture the runnable scheduled so we can invoke it when we want to - List channelRefreshers = new ArrayList<>(); - ScheduledExecutorService scheduledExecutorService = - Mockito.mock(ScheduledExecutorService.class); - Answer extractChannelRefresher = - invocation -> { - channelRefreshers.add((Runnable) invocation.getArgument(0)); - return null; - }; - Mockito.doAnswer(extractChannelRefresher) - .when(scheduledExecutorService) - .schedule( - Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.eq(TimeUnit.MILLISECONDS)); - - FakeChannelFactory channelFactory = new FakeChannelFactory(Arrays.asList(underlyingChannels)); - ManagedChannel refreshingManagedChannel = - new RefreshingManagedChannel(channelFactory, scheduledExecutorService); - - // send a bunch of request to RefreshingManagedChannel, executor needs more than 1 thread to - // test out concurrency - ExecutorService executor = Executors.newFixedThreadPool(10); - - // channelCount - 1 because the last channel cannot be refreshed because the FakeChannelFactory - // has no more channel to create - for (int i = 0; i < channelCount - 1; i++) { - List> futures = new ArrayList<>(); - int requestCount = 100; - int whenToRefresh = r.nextInt(requestCount); - for (int j = 0; j < requestCount; j++) { - Runnable createNewCall = - () -> { - // create a new call and send message on refreshingManagedChannel - ClientCall call = - refreshingManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - @SuppressWarnings("unchecked") - ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); - call.start(listener, new Metadata()); - call.sendMessage("message"); - }; - futures.add(executor.submit(createNewCall)); - // at the randomly chosen point, refresh the channel - if (j == whenToRefresh) { - futures.add(executor.submit(channelRefreshers.get(i))); - } - } - for (Future future : futures) { - future.get(); - } - Mockito.verify(underlyingChannels[i], Mockito.atLeastOnce()).shutdown(); - Mockito.verify(underlyingChannels[i + 1], Mockito.never()).shutdown(); - } - } -} diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/SafeShutdownManagedChannelTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/SafeShutdownManagedChannelTest.java deleted file mode 100644 index 91b16dcbf..000000000 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/SafeShutdownManagedChannelTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google LLC nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.api.gax.grpc; - -import com.google.api.gax.grpc.testing.FakeMethodDescriptor; -import io.grpc.CallOptions; -import io.grpc.ClientCall; -import io.grpc.ManagedChannel; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.Status; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; - -@RunWith(JUnit4.class) -public class SafeShutdownManagedChannelTest { - // call should be allowed to complete and the channel should not shutdown - @Test - public void callShouldCompleteAfterCreation() { - final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); - - SafeShutdownManagedChannel safeShutdownManagedChannel = - new SafeShutdownManagedChannel(underlyingChannel); - - // create a mock call when new call comes to the underlying channel - MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); - MockClientCall spyClientCall = Mockito.spy(mockClientCall); - Mockito.when( - underlyingChannel.newCall( - Mockito.>any(), Mockito.any(CallOptions.class))) - .thenReturn(spyClientCall); - - Answer verifyChannelNotShutdown = - invocation -> { - Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); - return invocation.callRealMethod(); - }; - - // verify that underlying channel is not shutdown when clientCall is still sending message - Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); - - // create a new call on safeShutdownManagedChannel - @SuppressWarnings("unchecked") - ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); - ClientCall call = - safeShutdownManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - - safeShutdownManagedChannel.shutdownSafely(); - // shutdown is not called because there is still an outstanding call, even if it hasn't started - Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); - - // start clientCall - call.start(listener, new Metadata()); - // send message and end the call - call.sendMessage("message"); - // shutdown is called because the outstanding call has completed - Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); - } - - // call should be allowed to complete and the channel should not shutdown - @Test - public void callShouldCompleteAfterStarted() { - final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); - - SafeShutdownManagedChannel safeShutdownManagedChannel = - new SafeShutdownManagedChannel(underlyingChannel); - - // create a mock call when new call comes to the underlying channel - MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); - MockClientCall spyClientCall = Mockito.spy(mockClientCall); - Mockito.when( - underlyingChannel.newCall( - Mockito.>any(), Mockito.any(CallOptions.class))) - .thenReturn(spyClientCall); - - Answer verifyChannelNotShutdown = - invocation -> { - Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); - return invocation.callRealMethod(); - }; - - // verify that underlying channel is not shutdown when clientCall is still sending message - Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); - - // create a new call on safeShutdownManagedChannel - @SuppressWarnings("unchecked") - ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); - ClientCall call = - safeShutdownManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - - // start clientCall - call.start(listener, new Metadata()); - safeShutdownManagedChannel.shutdownSafely(); - // shutdown is not called because there is still an outstanding call - Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); - // send message and end the call - call.sendMessage("message"); - // shutdown is called because the outstanding call has completed - Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); - } - - // Channel should shutdown after a refresh all the calls have completed - @Test - public void channelShouldShutdown() { - final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); - - SafeShutdownManagedChannel safeShutdownManagedChannel = - new SafeShutdownManagedChannel(underlyingChannel); - - // create a mock call when new call comes to the underlying channel - MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); - MockClientCall spyClientCall = Mockito.spy(mockClientCall); - Mockito.when( - underlyingChannel.newCall( - Mockito.>any(), Mockito.any(CallOptions.class))) - .thenReturn(spyClientCall); - - Answer verifyChannelNotShutdown = - invocation -> { - Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); - return invocation.callRealMethod(); - }; - - // verify that underlying channel is not shutdown when clientCall is still sending message - Mockito.doAnswer(verifyChannelNotShutdown).when(spyClientCall).sendMessage(Mockito.anyString()); - - // create a new call on safeShutdownManagedChannel - @SuppressWarnings("unchecked") - ClientCall.Listener listener = Mockito.mock(ClientCall.Listener.class); - ClientCall call = - safeShutdownManagedChannel.newCall( - FakeMethodDescriptor.create(), CallOptions.DEFAULT); - - // start clientCall - call.start(listener, new Metadata()); - // send message and end the call - call.sendMessage("message"); - // shutdown is not called because it has not been shutdown yet - Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); - safeShutdownManagedChannel.shutdownSafely(); - // shutdown is called because the outstanding call has completed - Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); - } -} From 3a501e0de709fc1055ac17768729fd89cfcc2287 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 22 Nov 2021 17:01:37 -0500 Subject: [PATCH 04/17] fix test --- .../java/com/google/api/gax/grpc/GrpcClientCallsTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java index ef0d58ce9..440d57209 100644 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java +++ b/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java @@ -30,6 +30,7 @@ package com.google.api.gax.grpc; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.verify; import com.google.api.gax.grpc.testing.FakeChannelFactory; import com.google.api.gax.grpc.testing.FakeServiceGrpc; @@ -83,8 +84,8 @@ public void testAffinity() throws IOException { ClientCall gotCallC = GrpcClientCalls.newCall(descriptor, context.withChannelAffinity(1)); - assertThat(gotCallA).isSameInstanceAs(gotCallB); - assertThat(gotCallA).isNotSameInstanceAs(gotCallC); + verify(channel0, Mockito.times(2)).newCall(Mockito.eq(descriptor), Mockito.any()); + verify(channel1, Mockito.times(1)).newCall(Mockito.eq(descriptor), Mockito.any()); } @Test From 6cb0964c470fdac1f11feb5bbcc7b60f7144949b Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 9 Dec 2021 16:19:08 -0500 Subject: [PATCH 05/17] address feedback --- .../com/google/api/gax/grpc/ChannelPool.java | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java index 2c73b1c54..fabb91503 100644 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java +++ b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java @@ -75,8 +75,7 @@ class ChannelPool extends ManagedChannel { private static final Duration REFRESH_PERIOD = Duration.ofMinutes(50); private static final double JITTER_PERCENTAGE = 0.15; - // A copy on write list of child channels. - private final AtomicReference> entries = new AtomicReference<>(); + private final AtomicReference> entries = new AtomicReference<>(); private final AtomicInteger indexTicker = new AtomicInteger(); private final String authority; // if set, ChannelPool will manage the life cycle of channelRefreshExecutorService @@ -269,11 +268,8 @@ private ScheduledFuture scheduleNextRefresh() { long delay = jitter + delayPeriod; return channelRefreshExecutorService.schedule( () -> { - try { - refresh(); - } finally { - scheduleNextRefresh(); - } + scheduleNextRefresh(); + refresh(); }, delay, TimeUnit.MILLISECONDS); @@ -300,31 +296,48 @@ void refresh() { LOG.log(Level.WARNING, "Failed to refresh channel, leaving old channel", e); } } - entries.set(ImmutableList.copyOf(newEntries)); + + ImmutableList replacedEntries = entries.getAndSet(ImmutableList.copyOf(newEntries)); + + // In the unlikely case that the list was modified while the new channels were being created, + // shutdown the unexpected channels. + for (Entry e : replacedEntries) { + if (!newEntries.contains(e) && !removedEntries.contains(e)) { + removedEntries.add(e); + } + } removedEntries.forEach(Entry::requestShutdown); } /** - * Get and retain a Channel Entry. The returned will have its rpc count incremented, preventing it - * from getting recycled. + * Get and retain a Channel Entry. The returned Entry will have its rpc count incremented, + * preventing it from getting recycled. */ Entry getRetainedEntry(int affinity) { + // The maximum number of concurrent calls to this method for any given time span is at most 2, + // so the loop can actually be 2 times. But going for 5 times for a safety margin for potential + // code evolving for (int i = 0; i < 5; i++) { Entry entry = getEntry(affinity); if (entry.retain()) { return entry; } } - throw new IllegalStateException("Failed to retain a channel"); + // It is unlikely to reach here unless the pool code evolves to increase the maximum possible + // concurrent calls to this method. If it does, this is a bug in the channel pool implementation + // the number of retries above should be greater than the number of contending maintenance + // tasks. + throw new IllegalStateException("Bug: failed to retain a channel"); } /** * Returns one of the channels managed by this pool. The pool continues to "own" the channel, and * the caller should not shut it down. * - * @param affinity Two calls to this method with the same affinity returns the same channel. The - * reverse is not true: Two calls with different affinities might return the same channel. - * However, the implementation should attempt to spread load evenly. + * @param affinity Two calls to this method with the same affinity returns the same channel most + * of the time, if the channel pool was refreshed since the last call, a new channel will be + * returned. The reverse is not true: Two calls with different affinities might return the + * same channel. However, the implementation should attempt to spread load evenly. */ private Entry getEntry(int affinity) { List localEntries = entries.get(); @@ -378,10 +391,12 @@ private boolean retain() { private void release() { int newCount = outstandingRpcs.decrementAndGet(); if (newCount < 0) { - LOG.log(Level.SEVERE, "Reference count is negative!: " + newCount); + throw new IllegalStateException("Bug: reference count is negative!: " + newCount); } - if (newCount == 0 && shutdownRequested.get()) { + // Must check outstandingRpcs after shutdownRequested (in reverse order of retain()) to ensure + // mutual exclusion. + if (shutdownRequested.get() && outstandingRpcs.get() == 0) { shutdown(); } } From 8f6e39edc1f169e62e0ffa041c06b0860c39178b Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 23 Dec 2021 16:37:09 -0500 Subject: [PATCH 06/17] fix race condition on refresh() --- .../com/google/api/gax/grpc/ChannelPool.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java index 9f705eaaf..4ebc3782c 100644 --- a/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java +++ b/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java @@ -278,14 +278,11 @@ private ScheduledFuture scheduleNextRefresh() { */ @InternalApi("Visible for testing") void refresh() { - List localEntries = entries.get(); - ArrayList newEntries = new ArrayList<>(localEntries); - ArrayList removedEntries = new ArrayList<>(); + ArrayList newEntries = new ArrayList<>(entries.get()); - for (int i = 0; i < localEntries.size(); i++) { + for (int i = 0; i < newEntries.size(); i++) { try { - Entry removed = newEntries.set(i, new Entry(channelFactory.createSingleChannel())); - removedEntries.add(removed); + newEntries.set(i, new Entry(channelFactory.createSingleChannel())); } catch (IOException e) { LOG.log(Level.WARNING, "Failed to refresh channel, leaving old channel", e); } @@ -293,15 +290,13 @@ void refresh() { ImmutableList replacedEntries = entries.getAndSet(ImmutableList.copyOf(newEntries)); - // In the unlikely case that the list was modified while the new channels were being created, - // shutdown the unexpected channels. + // Shutdown the channels that were cycled out. This will either be the channels we just + // refreshed or in case of a race, the channels that the other thread set. for (Entry e : replacedEntries) { - if (!newEntries.contains(e) && !removedEntries.contains(e)) { - removedEntries.add(e); + if (!newEntries.contains(e)) { + e.requestShutdown(); } } - - removedEntries.forEach(Entry::requestShutdown); } /** From c93daf4ac892d0db324abb506ce42b9a90d2823f Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 23 Dec 2021 16:37:44 -0500 Subject: [PATCH 07/17] fix warnings in test --- .../google/api/gax/grpc/ChannelPoolTest.java | 79 +++++++------------ 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java index 53a1447b2..83de7e77e 100644 --- a/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java +++ b/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java @@ -57,7 +57,6 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @RunWith(JUnit4.class) @@ -129,13 +128,9 @@ public void ensureEvenDistribution() throws InterruptedException, IOException { channels[i] = Mockito.mock(ManagedChannel.class); Mockito.when(channels[i].newCall(methodDescriptor, callOptions)) .thenAnswer( - new Answer>() { - @Override - public ClientCall answer(InvocationOnMock invocationOnMock) - throws Throwable { - counts[index].incrementAndGet(); - return clientCall; - } + (ignored) -> { + counts[index].incrementAndGet(); + return clientCall; }); } @@ -148,12 +143,9 @@ public ClientCall answer(InvocationOnMock invocationOnMock) ExecutorService executor = Executors.newFixedThreadPool(numThreads); for (int i = 0; i < numThreads; i++) { executor.submit( - new Runnable() { - @Override - public void run() { - for (int j = 0; j < numPerThread; j++) { - pool.newCall(methodDescriptor, callOptions); - } + () -> { + for (int j = 0; j < numPerThread; j++) { + pool.newCall(methodDescriptor, callOptions); } }); } @@ -195,7 +187,7 @@ public void channelPrimerIsCalledPeriodically() throws IOException { Answer extractChannelRefresher = invocation -> { - channelRefreshers.add((Runnable) invocation.getArgument(0)); + channelRefreshers.add(invocation.getArgument(0)); return Mockito.mock(ScheduledFuture.class); }; @@ -234,7 +226,7 @@ public void channelPrimerIsCalledPeriodically() throws IOException { } // ---- - // call should be allowed to complete and the channel should not shutdown + // call should be allowed to complete and the channel should not be shutdown @Test public void callShouldCompleteAfterCreation() throws IOException { final ManagedChannel underlyingChannel = Mockito.mock(ManagedChannel.class); @@ -251,13 +243,10 @@ public void callShouldCompleteAfterCreation() throws IOException { Mockito.>any(), Mockito.any(CallOptions.class))) .thenReturn(spyClientCall); - Answer verifyChannelNotShutdown = - new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); - return invocation.callRealMethod(); - } + Answer