Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 71dd021

Browse files
kannanjgithubkannanjgithub
committed
Ext_proc stream failure should not cause data plane call failure in observability mode.
1 parent 68532f5 commit 71dd021

2 files changed

Lines changed: 80 additions & 4 deletions

File tree

‎xds/src/main/java/io/grpc/xds/ExternalProcessorServerInterceptor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ boolean markDataPlaneCallClosed() {
427427

428428
private void activateCall() {
429429
if ((extProcStreamState.get() == ExtProcStreamState.FAILED
430+
&& !config.getObservabilityMode()
430431
&& (!config.getFailureModeAllow() || bodyMessageSentToExtProc.get()))
431432
|| !dataPlaneCallState.compareAndSet(
432433
DataPlaneCallState.IDLE, DataPlaneCallState.ACTIVE)) {
@@ -647,7 +648,8 @@ public void onError(Throwable t) {
647648
synchronized (streamLock) {
648649
extProcClientCallRequestObserver = null;
649650
}
650-
if (config.getFailureModeAllow() && !bodyMessageSentToExtProc.get()) {
651+
if (config.getObservabilityMode()
652+
|| (config.getFailureModeAllow() && !bodyMessageSentToExtProc.get())) {
651653
handleFailOpen();
652654
} else {
653655
rawCall.close(Status.INTERNAL.withDescription("External processor stream failed").withCause(t), new Metadata());
@@ -785,7 +787,8 @@ private void internalOnError(Throwable t) {
785787
}
786788
}
787789
expectedResponses.clear();
788-
if (config.getFailureModeAllow() && !bodyMessageSentToExtProc.get()) {
790+
if (config.getObservabilityMode()
791+
|| (config.getFailureModeAllow() && !bodyMessageSentToExtProc.get())) {
789792
handleFailOpen();
790793
} else {
791794
rawCall.close(Status.INTERNAL.withDescription("External processor stream failed").withCause(t), new Metadata());
@@ -964,6 +967,7 @@ public void sendMessage(InputStream message) {
964967
public void close(Status status, Metadata trailers) {
965968
serverTrailersStartNanos = System.nanoTime();
966969
if (isExtProcStreamFailed()
970+
&& !config.getObservabilityMode()
967971
&& (!config.getFailureModeAllow() || bodyMessageSentToExtProc.get())) {
968972
if (markDataPlaneCallClosed()) {
969973
proceedWithClose(Status.INTERNAL.withDescription("External processor stream failed").withCause(status.getCause()), new Metadata());

‎xds/src/test/java/io/grpc/xds/ExternalProcessorServerInterceptorTest.java‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6918,15 +6918,87 @@ public void onClose(Status status, Metadata trailers) {
69186918
clientCall.sendMessage(new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8)));
69196919
clientCall.halfClose();
69206920

6921-
// Verify stream failed
6922-
assertThat(extProcLatch.await(5, TimeUnit.SECONDS)).isTrue();
69236921
// Verify call completed and failed with INTERNAL status (not fail-open)
69246922
assertThat(callCompletedLatch.await(5, TimeUnit.SECONDS)).isTrue();
69256923
assertThat(callStarted.get()).isTrue();
69266924
assertThat(closedStatus.get().getCode()).isEqualTo(Status.Code.INTERNAL);
69276925
assertThat(closedStatus.get().getDescription()).contains("External processor stream failed");
69286926
}
69296927

6928+
@Test
6929+
public void givenObservabilityTrue_whenExtProcStreamFails_thenCallContinues() throws Exception {
6930+
ExternalProcessor proto = createBaseProto(extProcServerName)
6931+
.setObservabilityMode(true)
6932+
.build();
6933+
ConfigOrError<ExternalProcessorFilterConfig> configOrError =
6934+
provider.parseFilterConfig(Any.pack(proto), filterContext);
6935+
assertThat(configOrError.errorDetail).isNull();
6936+
ExternalProcessorFilterConfig filterConfig = configOrError.config;
6937+
6938+
ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl =
6939+
new ExternalProcessorGrpc.ExternalProcessorImplBase() {
6940+
@Override
6941+
public StreamObserver<ProcessingRequest> process(
6942+
StreamObserver<ProcessingResponse> responseObserver) {
6943+
responseObserver.onError(Status.UNAVAILABLE.withDescription("ExtProc down").asRuntimeException());
6944+
return new StreamObserver<ProcessingRequest>() {
6945+
@Override public void onNext(ProcessingRequest request) {}
6946+
@Override public void onError(Throwable t) {}
6947+
@Override public void onCompleted() {}
6948+
};
6949+
}
6950+
};
6951+
6952+
grpcCleanup.register(InProcessServerBuilder.forName(extProcServerName)
6953+
.addService(extProcImpl)
6954+
.directExecutor()
6955+
.build().start());
6956+
6957+
CachedChannelManager channelManager = new CachedChannelManager(config -> {
6958+
return grpcCleanup.register(
6959+
InProcessChannelBuilder.forName(extProcServerName)
6960+
.directExecutor()
6961+
.build());
6962+
});
6963+
6964+
ExternalProcessorServerInterceptor interceptor = new ExternalProcessorServerInterceptor(
6965+
filterConfig, channelManager, FAKE_CONTEXT);
6966+
6967+
final AtomicBoolean callStarted = new AtomicBoolean(false);
6968+
dataPlaneHandler = new DataPlaneServiceHandler() {
6969+
@Override
6970+
public void sayHello(InputStream request, StreamObserver<InputStream> responseObserver) {
6971+
callStarted.set(true);
6972+
responseObserver.onNext(request);
6973+
responseObserver.onCompleted();
6974+
}
6975+
};
6976+
6977+
startDataPlane(interceptor);
6978+
6979+
io.grpc.ClientCall<InputStream, InputStream> clientCall = dataPlaneChannel.newCall(
6980+
METHOD_SAY_HELLO_RAW, io.grpc.CallOptions.DEFAULT);
6981+
6982+
final CountDownLatch callCompletedLatch = new CountDownLatch(1);
6983+
final AtomicReference<Status> closedStatus = new AtomicReference<>();
6984+
clientCall.start(new io.grpc.ClientCall.Listener<InputStream>() {
6985+
@Override
6986+
public void onClose(Status status, Metadata trailers) {
6987+
closedStatus.set(status);
6988+
callCompletedLatch.countDown();
6989+
}
6990+
}, new Metadata());
6991+
6992+
clientCall.request(1);
6993+
clientCall.sendMessage(new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8)));
6994+
clientCall.halfClose();
6995+
6996+
assertThat(callCompletedLatch.await(5, TimeUnit.SECONDS)).isTrue();
6997+
// Verify call completed and succeeded with OK status even though stream failed
6998+
assertThat(callStarted.get()).isTrue();
6999+
assertThat(closedStatus.get().getCode()).isEqualTo(Status.Code.OK);
7000+
}
7001+
69307002
// ============================================================================
69317003
// Category 16: Immediate Response Handling
69327004
// ============================================================================

0 commit comments

Comments
 (0)