Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit fe608fe

Browse files
rahul2393olavloite
rahul2393
and
olavloite
authored
fix(spanner): avoid double grpc-gcp wrapping for directpath fallback (#13155)
Fixes DirectPath fallback channel construction so each path gets only one grpc-gcp layer. Before: ``` GcpManagedChannel -> ChannelRef -> inner GcpManagedChannel -> ChannelRef ``` After: ``` GcpFallbackChannel -> DirectPath GcpManagedChannel -> CloudPath GcpManagedChannel ``` Impact: ``` 432 ChannelRef -> 48 ChannelRef 54 GcpManagedChannel -> 6 GcpManagedChannel ``` Internal reference: go/grpc-gcp-directpath-fixes --------- Co-authored-by: Knut Olav Løite <[email protected]>
1 parent e03efe0 commit fe608fe

3 files changed

Lines changed: 336 additions & 31 deletions

File tree

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -372,16 +372,18 @@ public GapicSpannerRpc(final SpannerOptions options) {
372372
options, headerProviderWithUserAgent, isEnableDirectAccess);
373373
GrpcGcpEndpointChannelConfigurator endpointChannelConfigurator =
374374
createGrpcGcpEndpointChannelConfigurator(defaultChannelProviderBuilder, options);
375-
maybeEnableGrpcGcpExtension(defaultChannelProviderBuilder, options);
376-
377-
if (options.getChannelProvider() == null
378-
&& isEnableDirectAccess
379-
&& options.isEnableGcpFallback()) {
375+
boolean useGcpFallback =
376+
options.getChannelProvider() == null
377+
&& isEnableDirectAccess
378+
&& options.isEnableGcpFallback();
379+
if (useGcpFallback) {
380380
setupGcpFallback(
381381
defaultChannelProviderBuilder,
382382
options,
383383
headerProviderWithUserAgent,
384384
credentialsProvider);
385+
} else {
386+
maybeEnableGrpcGcpExtension(defaultChannelProviderBuilder, options);
385387
}
386388

387389
boolean enableLocationApi = options.isEnableLocationApi();
@@ -656,8 +658,11 @@ private void setupGcpFallback(
656658
final HeaderProvider headerProviderWithUserAgent,
657659
final CredentialsProvider credentialsProvider) {
658660
InstantiatingGrpcChannelProvider.Builder cloudPathProviderBuilder =
659-
createChannelProviderBuilder(
661+
createBaseChannelProviderBuilder(
660662
options, headerProviderWithUserAgent, /* isEnableDirectAccess= */ false);
663+
if (options.isGrpcGcpExtensionEnabled()) {
664+
cloudPathProviderBuilder.setPoolSize(1);
665+
}
661666

662667
InstantiatingGrpcChannelProvider cloudPathProvider = cloudPathProviderBuilder.build();
663668
ManagedChannelBuilder cloudPathBuilder;
@@ -689,29 +694,34 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
689694

690695
final ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> existingConfigurator =
691696
defaultChannelProviderBuilder.getChannelConfigurator();
697+
if (options.isGrpcGcpExtensionEnabled()) {
698+
defaultChannelProviderBuilder.setPoolSize(1);
699+
}
692700
defaultChannelProviderBuilder.setChannelConfigurator(
693701
directPathBuilder -> {
694702
ManagedChannelBuilder builder = directPathBuilder;
695703
if (existingConfigurator != null) {
696704
builder = existingConfigurator.apply(builder);
697705
}
698706

699-
String jsonApiConfig = parseGrpcGcpApiConfig();
700-
GcpManagedChannelOptions gcpOptions = grpcGcpOptionsWithMetricsAndDcp(options);
701-
if (gcpOptions == null) {
702-
gcpOptions = GcpManagedChannelOptions.newBuilder().build();
707+
ManagedChannelBuilder<?> primaryBuilder = builder;
708+
ManagedChannelBuilder<?> fallbackBuilder = cloudPathBuilder;
709+
if (options.isGrpcGcpExtensionEnabled()) {
710+
String jsonApiConfig = parseGrpcGcpApiConfig();
711+
GcpManagedChannelOptions gcpOptions = grpcGcpOptionsWithMetricsAndDcp(options);
712+
if (gcpOptions == null) {
713+
gcpOptions = GcpManagedChannelOptions.newBuilder().build();
714+
}
715+
primaryBuilder =
716+
GcpManagedChannelBuilder.forDelegateBuilder(builder)
717+
.withApiConfigJsonString(jsonApiConfig)
718+
.withOptions(gcpOptions);
719+
fallbackBuilder =
720+
GcpManagedChannelBuilder.forDelegateBuilder(cloudPathBuilder)
721+
.withApiConfigJsonString(jsonApiConfig)
722+
.withOptions(gcpOptions);
703723
}
704724

705-
GcpManagedChannelBuilder primaryGcpBuilder =
706-
GcpManagedChannelBuilder.forDelegateBuilder(builder)
707-
.withApiConfigJsonString(jsonApiConfig)
708-
.withOptions(gcpOptions);
709-
710-
GcpManagedChannelBuilder fallbackGcpBuilder =
711-
GcpManagedChannelBuilder.forDelegateBuilder(cloudPathBuilder)
712-
.withApiConfigJsonString(jsonApiConfig)
713-
.withOptions(gcpOptions);
714-
715725
GcpFallbackOpenTelemetry fallbackTelemetry =
716726
GcpFallbackOpenTelemetry.newBuilder()
717727
.withSdk(getFallbackOpenTelemetry(options))
@@ -720,9 +730,7 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
720730
.build();
721731

722732
return new FallbackChannelBuilder(
723-
primaryGcpBuilder,
724-
fallbackGcpBuilder,
725-
createFallbackChannelOptions(fallbackTelemetry, 1));
733+
primaryBuilder, fallbackBuilder, createFallbackChannelOptions(fallbackTelemetry, 1));
726734
});
727735
}
728736

@@ -2595,15 +2603,15 @@ private static class FallbackChannelBuilder
25952603
extends ForwardingChannelBuilder2<FallbackChannelBuilder> {
25962604
private final GcpFallbackChannelOptions options;
25972605

2598-
private final GcpManagedChannelBuilder primaryGcpBuilder;
2599-
private final GcpManagedChannelBuilder fallbackGcpBuilder;
2606+
private final ManagedChannelBuilder<?> primaryBuilder;
2607+
private final ManagedChannelBuilder<?> fallbackBuilder;
26002608

26012609
private FallbackChannelBuilder(
2602-
GcpManagedChannelBuilder primary,
2603-
GcpManagedChannelBuilder fallback,
2610+
ManagedChannelBuilder<?> primary,
2611+
ManagedChannelBuilder<?> fallback,
26042612
GcpFallbackChannelOptions options) {
2605-
this.primaryGcpBuilder = primary;
2606-
this.fallbackGcpBuilder = fallback;
2613+
this.primaryBuilder = primary;
2614+
this.fallbackBuilder = fallback;
26072615
this.options = options;
26082616
}
26092617

@@ -2613,7 +2621,7 @@ private FallbackChannelBuilder(
26132621
*/
26142622
@Override
26152623
protected ManagedChannelBuilder<?> delegate() {
2616-
return primaryGcpBuilder;
2624+
return primaryBuilder;
26172625
}
26182626

26192627
/**
@@ -2622,7 +2630,7 @@ protected ManagedChannelBuilder<?> delegate() {
26222630
*/
26232631
@Override
26242632
public ManagedChannel build() {
2625-
return new GcpFallbackChannel(options, primaryGcpBuilder, fallbackGcpBuilder);
2633+
return new GcpFallbackChannel(options, primaryBuilder, fallbackBuilder);
26262634
}
26272635
}
26282636
}

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcConnectionTest.java‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.spi.v1;
18+
19+
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assume.assumeTrue;
21+
22+
import com.google.cloud.NoCredentials;
23+
import com.google.cloud.spanner.MockSpannerServiceImpl;
24+
import com.google.cloud.spanner.SpannerOptions;
25+
import com.google.common.base.Stopwatch;
26+
import io.grpc.Attributes;
27+
import io.grpc.ManagedChannelBuilder;
28+
import io.grpc.Server;
29+
import io.grpc.ServerTransportFilter;
30+
import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
31+
import java.net.InetSocketAddress;
32+
import java.util.concurrent.TimeUnit;
33+
import java.util.concurrent.atomic.AtomicInteger;
34+
import org.junit.After;
35+
import org.junit.Before;
36+
import org.junit.Test;
37+
import org.junit.runner.RunWith;
38+
import org.junit.runners.JUnit4;
39+
40+
@RunWith(JUnit4.class)
41+
public class GapicSpannerRpcConnectionTest {
42+
43+
private static MockSpannerServiceImpl mockSpanner;
44+
private static Server server;
45+
private static InetSocketAddress address;
46+
47+
private final AtomicInteger activeNetworkConnections = new AtomicInteger(0);
48+
49+
private final ServerTransportFilter connectionCounterFilter =
50+
new ServerTransportFilter() {
51+
@Override
52+
public Attributes transportReady(Attributes transportAttrs) {
53+
activeNetworkConnections.incrementAndGet();
54+
return super.transportReady(transportAttrs);
55+
}
56+
57+
@Override
58+
public void transportTerminated(Attributes transportAttrs) {
59+
activeNetworkConnections.decrementAndGet();
60+
super.transportTerminated(transportAttrs);
61+
}
62+
};
63+
64+
@Before
65+
public void startServer() throws Exception {
66+
mockSpanner = new MockSpannerServiceImpl();
67+
mockSpanner.setAbortProbability(0.0D);
68+
69+
address = new InetSocketAddress("localhost", 0);
70+
server =
71+
NettyServerBuilder.forAddress(address)
72+
.addService(mockSpanner)
73+
.addTransportFilter(connectionCounterFilter)
74+
.build()
75+
.start();
76+
activeNetworkConnections.set(0);
77+
}
78+
79+
@After
80+
public void reset() throws InterruptedException {
81+
if (mockSpanner != null) {
82+
mockSpanner.reset();
83+
}
84+
if (server != null) {
85+
server.shutdown();
86+
server.awaitTermination();
87+
}
88+
}
89+
90+
private SpannerOptions.Builder createDirectPathFallbackOptions() {
91+
String endpoint = address.getHostString() + ":" + server.getPort();
92+
return SpannerOptions.newBuilder()
93+
.setProjectId("test-project")
94+
.setChannelConfigurator(ManagedChannelBuilder::usePlaintext)
95+
.setEnableDirectAccess(true)
96+
.setHost("http://" + endpoint)
97+
.setCredentials(NoCredentials.getInstance());
98+
}
99+
100+
@Test
101+
public void testDirectPathFallbackCreatesExactlyFourPhysicalSockets() {
102+
SpannerOptions.useEnvironment(new SpannerOptions.SpannerEnvironment() {});
103+
GapicSpannerRpc rpc = null;
104+
try {
105+
SpannerOptions options = createDirectPathFallbackOptions().build();
106+
assumeTrue(
107+
"GCP fallback must be enabled for this DirectPath fallback test",
108+
options.isEnableGcpFallback());
109+
110+
activeNetworkConnections.set(0);
111+
rpc = new GapicSpannerRpc(options);
112+
113+
// Poll active loopback connections for up to 1000ms with an aggressive 1ms wait
114+
Stopwatch watch = Stopwatch.createStarted();
115+
while (activeNetworkConnections.get() < 48 && watch.elapsed(TimeUnit.MILLISECONDS) < 1000L) {
116+
try {
117+
Thread.sleep(1L);
118+
} catch (InterruptedException ignored) {
119+
}
120+
}
121+
122+
// Sleep for an extra 5ms after seeing 48 connections (or hitting timeout) to
123+
// ensure we catch any additional connections that are created.
124+
try {
125+
Thread.sleep(5L);
126+
} catch (InterruptedException ignored) {
127+
}
128+
129+
// Assert that the Spanner client stubs eagerly construct exactly 3 fallback channels:
130+
// 1. Shared pool for the Data client and PartitionedDML client stubs
131+
// 2. Dedicated pool for the InstanceAdmin client stub
132+
// 3. Dedicated pool for the DatabaseAdmin client stub
133+
// Each fallback channel contains a primary and fallback pool (totaling 6
134+
// GcpManagedChannel pools).
135+
// Since the default pool size is 8 channels when gRPC-GCP is enabled, they eagerly
136+
// establish exactly 48 physical Loopback TCP connection sockets (6 pools of size 8).
137+
assertEquals(48, activeNetworkConnections.get());
138+
} finally {
139+
if (rpc != null) {
140+
rpc.shutdown();
141+
}
142+
SpannerOptions.useDefaultEnvironment();
143+
}
144+
}
145+
}

0 commit comments

Comments
 (0)