From b3eaef1bf8c9fd2cb3b72bed57bf6c7aea428190 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Mon, 26 Jul 2021 09:47:09 -0700 Subject: [PATCH 01/87] Bump version --- grpc-gcp/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grpc-gcp/build.gradle b/grpc-gcp/build.gradle index 5d7e3587..367379e4 100644 --- a/grpc-gcp/build.gradle +++ b/grpc-gcp/build.gradle @@ -15,7 +15,7 @@ repositories { mavenLocal() } -version = '1.1.0' +version = '1.1.1-SNAPSHOT' group = 'com.google.cloud' description = 'GRPC-GCP-Extension Java' sourceCompatibility = '1.8' From b05da2276a2182755e6821e81b8d4d4967edc83a Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 9 Aug 2021 13:34:41 -0700 Subject: [PATCH 02/87] Include fs.open in the totla elapsed time --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 16bc436c..69cbb486 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -123,8 +123,8 @@ private void makeMediaRequest(ResultTable results) throws IOException { ByteBuffer buff = ByteBuffer.allocate(size); for (int i = 0; i < args.calls; i++) { - ReadableByteChannel readChannel = gcsfs.open(uri); long start = System.currentTimeMillis(); + ReadableByteChannel readChannel = gcsfs.open(uri); readChannel.read(buff); long dur = System.currentTimeMillis() - start; if (buff.remaining() > 0) { @@ -153,9 +153,9 @@ private void makeWriteRequest(ResultTable results, int idx) throws IOException, URI uri = URI.create("gs://" + args.bkt + "/" + args.obj + "_" + idx); for (int i = 0; i < args.calls; i++) { + long start = System.currentTimeMillis(); WritableByteChannel writeChannel = gcsfs.create(uri); ByteBuffer buff = ByteBuffer.wrap(randBytes); - long start = System.currentTimeMillis(); writeChannel.write(buff); writeChannel.close(); // write operation is async, need to call close() to wait for finish. From 8d64709be4913eae3f5cd839fc4111a4428db600 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 19 Aug 2021 10:00:16 -0700 Subject: [PATCH 03/87] Sync with hadoop-connector --- .../grpc/gcs/ZeroCopyMessageMarshaller.java | 93 ++++++++++--------- .../io/grpc/gcs/ZeroCopyReadinessChecker.java | 23 +++-- 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java index 82d403b7..5a79273e 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java @@ -10,11 +10,10 @@ import io.grpc.HasByteBuffer; import io.grpc.KnownLength; import io.grpc.MethodDescriptor.PrototypeMarshaller; -import io.grpc.protobuf.lite.ProtoLiteUtils; import io.grpc.Status; -import io.grpc.MethodDescriptor; -import java.io.InputStream; +import io.grpc.protobuf.lite.ProtoLiteUtils; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -22,76 +21,80 @@ import java.util.List; import java.util.Map; -// Custom gRPC marshaller to use zero memory copy feature of gRPC when deserializing messages. -// This achieves zero-copy by deserializing proto messages pointing to the buffers in the input -// stream to avoid memory copy so stream should live as long as the message can be referenced. -// Hence, it exposes the input stream to applications (through popStream) and applications are -// responsible to close it when it's no longer needed. Otherwise, it'd cause memory leak. +/** + * Custom gRPC marshaller to use zero memory copy feature of gRPC when deserializing messages. This + * achieves zero-copy by deserializing proto messages pointing to the buffers in the input stream to + * avoid memory copy so stream should live as long as the message can be referenced. Hence, it + * exposes the input stream to applications (through popStream) and applications are responsible to + * close it when it's no longer needed. Otherwise, it'd cause memory leak. + */ class ZeroCopyMessageMarshaller implements PrototypeMarshaller { - private Map unclosedStreams = Collections.synchronizedMap(new IdentityHashMap<>()); + private Map unclosedStreams = + Collections.synchronizedMap(new IdentityHashMap<>()); private final Parser parser; - private final PrototypeMarshaller baseMarshaller; + private final PrototypeMarshaller marshaller; ZeroCopyMessageMarshaller(T defaultInstance) { parser = (Parser) defaultInstance.getParserForType(); - baseMarshaller = (PrototypeMarshaller) ProtoLiteUtils.marshaller(defaultInstance); + marshaller = (PrototypeMarshaller) ProtoLiteUtils.marshaller(defaultInstance); } @Override public Class getMessageClass() { - return baseMarshaller.getMessageClass(); + return marshaller.getMessageClass(); } @Override public T getMessagePrototype() { - return baseMarshaller.getMessagePrototype(); + return marshaller.getMessagePrototype(); } @Override public InputStream stream(T value) { - return baseMarshaller.stream(value); + return marshaller.stream(value); } @Override public T parse(InputStream stream) { - CodedInputStream cis = null; try { - if (stream instanceof KnownLength) { + if (stream instanceof KnownLength + && stream instanceof Detachable + && stream instanceof HasByteBuffer + && ((HasByteBuffer) stream).byteBufferSupported()) { int size = stream.available(); - if (stream instanceof Detachable && ((HasByteBuffer) stream).byteBufferSupported()) { - // Stream is now detached here and should be closed later. - stream = ((Detachable) stream).detach(); - // This mark call is to keep buffer while traversing buffers using skip. - stream.mark(size); - List byteStrings = new ArrayList<>(); - while (stream.available() != 0) { - ByteBuffer buffer = ((HasByteBuffer) stream).getByteBuffer(); - byteStrings.add(UnsafeByteOperations.unsafeWrap(buffer)); - stream.skip(buffer.remaining()); - } - stream.reset(); - cis = ByteString.copyFrom(byteStrings).newCodedInput(); - cis.enableAliasing(true); - cis.setSizeLimit(Integer.MAX_VALUE); + // Stream is now detached here and should be closed later. + stream = ((Detachable) stream).detach(); + // This mark call is to keep buffer while traversing buffers using skip. + stream.mark(size); + List byteStrings = new ArrayList<>(); + while (stream.available() != 0) { + ByteBuffer buffer = ((HasByteBuffer) stream).getByteBuffer(); + byteStrings.add(UnsafeByteOperations.unsafeWrap(buffer)); + stream.skip(buffer.remaining()); } + stream.reset(); + CodedInputStream codedInputStream = ByteString.copyFrom(byteStrings).newCodedInput(); + codedInputStream.enableAliasing(true); + codedInputStream.setSizeLimit(Integer.MAX_VALUE); + // fast path (no memory copy) + T message; + try { + message = parseFrom(codedInputStream); + } catch (InvalidProtocolBufferException ipbe) { + stream.close(); + throw Status.INTERNAL + .withDescription("Invalid protobuf byte sequence") + .withCause(ipbe) + .asRuntimeException(); + } + unclosedStreams.put(message, stream); + return message; } } catch (IOException e) { throw new RuntimeException(e); } - if (cis != null) { - // fast path (no memory copy) - T message; - try { - message = parseFrom(cis); - } catch (InvalidProtocolBufferException ipbe) { - throw Status.INTERNAL.withDescription("Invalid protobuf byte sequence").withCause(ipbe).asRuntimeException(); - } - unclosedStreams.put(message, stream); - return message; - } else { - // slow path - return baseMarshaller.parse(stream); - } + // slow path + return marshaller.parse(stream); } private T parseFrom(CodedInputStream stream) throws InvalidProtocolBufferException { diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyReadinessChecker.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyReadinessChecker.java index 825545a8..ebecd233 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyReadinessChecker.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyReadinessChecker.java @@ -1,12 +1,15 @@ package io.grpc.gcs; +import com.google.common.flogger.GoogleLogger; import com.google.protobuf.MessageLite; import io.grpc.KnownLength; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.security.Provider; -public class ZeroCopyReadinessChecker { +/** + * Checker to test whether a zero-copy masharller is available from the versions of gRPC and + * Protobuf. + */ +class ZeroCopyReadinessChecker { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); private static final boolean isZeroCopyReady; static { @@ -18,22 +21,26 @@ public class ZeroCopyReadinessChecker { // done indirectly to handle the case where gRPC is being shaded in a // different package. String knownLengthClassName = KnownLength.class.getName(); - String detachableClassName = knownLengthClassName.substring(0, knownLengthClassName.lastIndexOf('.') + 1) - + "Detachable"; + String detachableClassName = + knownLengthClassName.substring(0, knownLengthClassName.lastIndexOf('.') + 1) + + "Detachable"; Class detachableClass = Class.forName(detachableClassName); detachableClassExists = (detachableClass != null); } catch (ClassNotFoundException ex) { + logger.atFine().withCause(ex).log("io.grpc.Detachable not found"); } // Check whether com.google.protobuf.UnsafeByteOperations exists? boolean unsafeByteOperationsClassExists = false; try { // Same above String messageLiteClassName = MessageLite.class.getName(); - String unsafeByteOperationsClassName = messageLiteClassName.substring(0, - messageLiteClassName.lastIndexOf('.') + 1) + "UnsafeByteOperations"; + String unsafeByteOperationsClassName = + messageLiteClassName.substring(0, messageLiteClassName.lastIndexOf('.') + 1) + + "UnsafeByteOperations"; Class unsafeByteOperationsClass = Class.forName(unsafeByteOperationsClassName); unsafeByteOperationsClassExists = (unsafeByteOperationsClass != null); } catch (ClassNotFoundException ex) { + logger.atFine().withCause(ex).log("com.google.protobuf.UnsafeByteOperations not found"); } isZeroCopyReady = detachableClassExists && unsafeByteOperationsClassExists; } From cff95af726eb247ec785658658a9358015fc70bd Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 19 Aug 2021 12:33:38 -0700 Subject: [PATCH 04/87] Robust stream close --- .../src/main/java/io/grpc/gcs/GrpcClient.java | 23 ++++---- .../grpc/gcs/ZeroCopyMessageMarshaller.java | 54 ++++++++++--------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 7d1c1be2..83c28ffe 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -204,18 +204,21 @@ private void makeMediaRequest(ManagedChannel channel, ResultTable results) { try { while (true) { GetObjectMediaResponse res = resIterator.next(); - InputStream stream = getObjectMediaResponseMarshaller.popStream(res); - // Just copy to scratch memory to ensure its data is consumed. - ByteString content = res.getChecksummedData().getContent(); - content.copyTo(scratch, 0); // When zero-copy mashaller is used, the stream that backs GetObjectMediaResponse // should be closed when the mssage is no longed needed so that all buffers in the - // stream can be reclaimed. - if (stream != null) { - try { - stream.close(); - } catch (IOException e) { - throw new RuntimeException(e); + // stream can be reclaimed. If zero-copy is not used, stream will be null. + InputStream stream = getObjectMediaResponseMarshaller.popStream(res); + try { + // Just copy to scratch memory to ensure its data is consumed. + ByteString content = res.getChecksummedData().getContent(); + content.copyTo(scratch, 0); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } } } diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java index 5a79273e..a46dac05 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java @@ -63,32 +63,38 @@ public T parse(InputStream stream) { && ((HasByteBuffer) stream).byteBufferSupported()) { int size = stream.available(); // Stream is now detached here and should be closed later. - stream = ((Detachable) stream).detach(); - // This mark call is to keep buffer while traversing buffers using skip. - stream.mark(size); - List byteStrings = new ArrayList<>(); - while (stream.available() != 0) { - ByteBuffer buffer = ((HasByteBuffer) stream).getByteBuffer(); - byteStrings.add(UnsafeByteOperations.unsafeWrap(buffer)); - stream.skip(buffer.remaining()); - } - stream.reset(); - CodedInputStream codedInputStream = ByteString.copyFrom(byteStrings).newCodedInput(); - codedInputStream.enableAliasing(true); - codedInputStream.setSizeLimit(Integer.MAX_VALUE); - // fast path (no memory copy) - T message; + InputStream detachedStream = ((Detachable) stream).detach(); try { - message = parseFrom(codedInputStream); - } catch (InvalidProtocolBufferException ipbe) { - stream.close(); - throw Status.INTERNAL - .withDescription("Invalid protobuf byte sequence") - .withCause(ipbe) - .asRuntimeException(); + // This mark call is to keep buffer while traversing buffers using skip. + detachedStream.mark(size); + List byteStrings = new ArrayList<>(); + while (detachedStream.available() != 0) { + ByteBuffer buffer = ((HasByteBuffer) detachedStream).getByteBuffer(); + byteStrings.add(UnsafeByteOperations.unsafeWrap(buffer)); + stream.skip(buffer.remaining()); + } + detachedStream.reset(); + CodedInputStream codedInputStream = ByteString.copyFrom(byteStrings).newCodedInput(); + codedInputStream.enableAliasing(true); + codedInputStream.setSizeLimit(Integer.MAX_VALUE); + // fast path (no memory copy) + T message; + try { + message = parseFrom(codedInputStream); + } catch (InvalidProtocolBufferException ipbe) { + throw Status.INTERNAL + .withDescription("Invalid protobuf byte sequence") + .withCause(ipbe) + .asRuntimeException(); + } + unclosedStreams.put(message, detachedStream); + detachedStream = null; + return message; + } finally { + if (detachedStream != null) { + detachedStream.close(); + } } - unclosedStreams.put(message, stream); - return message; } } catch (IOException e) { throw new RuntimeException(e); From a72281dadb535197f0dfb8877c2bd94a12e46ca5 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 19 Aug 2021 13:09:10 -0700 Subject: [PATCH 05/87] fix bug --- .../src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java index a46dac05..fa15fd50 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ZeroCopyMessageMarshaller.java @@ -71,7 +71,7 @@ public T parse(InputStream stream) { while (detachedStream.available() != 0) { ByteBuffer buffer = ((HasByteBuffer) detachedStream).getByteBuffer(); byteStrings.add(UnsafeByteOperations.unsafeWrap(buffer)); - stream.skip(buffer.remaining()); + detachedStream.skip(buffer.remaining()); } detachedStream.reset(); CodedInputStream codedInputStream = ByteString.copyFrom(byteStrings).newCodedInput(); From e380ec54aa4f201efe1c6596ecae16e3885e4b9e Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 12:36:08 -0700 Subject: [PATCH 06/87] Use GCS/gRPC V2 API --- end2end-test-examples/gcs/build.gradle | 1 + .../src/main/java/io/grpc/gcs/GrpcClient.java | 91 +- .../proto/google/storage/v1/storage.proto | 1986 ----------------- .../google/storage/v1/storage_resources.proto | 793 ------- 4 files changed, 50 insertions(+), 2821 deletions(-) delete mode 100644 end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage.proto delete mode 100644 end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage_resources.proto diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 27d62e70..1d1a40da 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -29,6 +29,7 @@ dependencies { compile "io.grpc:grpc-alts:${grpcVersion}" compile "org.conscrypt:conscrypt-openjdk-uber:${conscryptVersion}" compile "com.google.protobuf:protobuf-java-util:${protobufVersion}" + compile "com.google.api.grpc:grpc-google-cloud-storage-v2:latest.release" compile "com.google.api.grpc:proto-google-iam-v1:latest.release" compile "com.google.api.grpc:proto-google-common-protos:latest.release" compile "net.sourceforge.argparse4j:argparse4j:0.8.1" diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 83c28ffe..3c5b757b 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -8,14 +8,16 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.google.storage.v1.ChecksummedData; -import com.google.google.storage.v1.GetObjectMediaRequest; -import com.google.google.storage.v1.GetObjectMediaResponse; -import com.google.google.storage.v1.InsertObjectRequest; -import com.google.google.storage.v1.InsertObjectSpec; -import com.google.google.storage.v1.Object; -import com.google.google.storage.v1.ServiceConstants.Values; -import com.google.google.storage.v1.StorageGrpc; +import com.google.storage.v2.ChecksummedData; +import com.google.storage.v2.ReadObjectRequest; +import com.google.storage.v2.ReadObjectResponse; +import com.google.storage.v2.WriteObjectRequest; +import com.google.storage.v2.WriteObjectSpec; +import com.google.storage.v2.WriteObjectResponse; +import com.google.storage.v2.Object; +import com.google.storage.v2.ServiceConstants.Values; +import com.google.storage.v2.StorageGrpc; +import com.google.storage.v2.StorageGrpc.StorageBlockingStub; import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -42,11 +44,11 @@ public class GrpcClient { private static final Logger logger = Logger.getLogger(GrpcClient.class.getName()); // ZeroCopy version of GetObjectMedia Method - private static final ZeroCopyMessageMarshaller getObjectMediaResponseMarshaller = - new ZeroCopyMessageMarshaller(GetObjectMediaResponse.getDefaultInstance()); - private static final MethodDescriptor getObjectMediaMethod = - StorageGrpc.getGetObjectMediaMethod() - .toBuilder().setResponseMarshaller(getObjectMediaResponseMarshaller) + private static final ZeroCopyMessageMarshaller ReadObjectResponseMarshaller = + new ZeroCopyMessageMarshaller(ReadObjectResponse.getDefaultInstance()); + private static final MethodDescriptor readObjectMethod = + StorageGrpc.getReadObjectMethod() + .toBuilder().setResponseMarshaller(ReadObjectResponseMarshaller) .build(); private final boolean useZeroCopy; @@ -55,6 +57,11 @@ public class GrpcClient { private GoogleCredentials creds; private static final String SCOPE = "https://www.googleapis.com/auth/cloud-platform"; + private static final String V2_BUCKET_NAME_PREFIX = "projects/_/buckets/"; + + private static String toV2BucketName(String v1BucketName) { + return V2_BUCKET_NAME_PREFIX + v1BucketName; + } public GrpcClient(Args args) throws IOException { this.args = args; @@ -123,10 +130,10 @@ public void startCalls(ResultTable results) throws InterruptedException { try { switch (args.method) { case METHOD_READ: - makeMediaRequest(channel, results); + makeReadObjectRequest(channel, results); break; case METHOD_RANDOM: - makeRandomMediaRequest(channel, results); + makeRandomReadRequest(channel, results); break; case METHOD_WRITE: makeInsertRequest(channel, results, 0); @@ -145,14 +152,14 @@ public void startCalls(ResultTable results) throws InterruptedException { case METHOD_READ: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> makeMediaRequest(this.channels[finalI], results); + Runnable task = () -> makeReadObjectRequest(this.channels[finalI], results); threadPoolExecutor.execute(task); } break; case METHOD_RANDOM: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> makeRandomMediaRequest(this.channels[finalI], results); + Runnable task = () -> makeRandomReadRequest(this.channels[finalI], results); threadPoolExecutor.execute(task); } break; @@ -181,7 +188,7 @@ public void startCalls(ResultTable results) throws InterruptedException { } } - private void makeMediaRequest(ManagedChannel channel, ResultTable results) { + private void makeReadObjectRequest(ManagedChannel channel, ResultTable results) { StorageGrpc.StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { @@ -189,25 +196,25 @@ private void makeMediaRequest(ManagedChannel channel, ResultTable results) { MoreCallCredentials.from(creds)); } - GetObjectMediaRequest mediaRequest = - GetObjectMediaRequest.newBuilder().setBucket(args.bkt).setObject(args.obj).build(); + ReadObjectRequest readRequest = + ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(args.obj).build(); byte[] scratch = new byte[4*1024*1024]; for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); - Iterator resIterator; + Iterator resIterator; if (useZeroCopy) { resIterator = io.grpc.stub.ClientCalls.blockingServerStreamingCall( - blockingStub.getChannel(), getObjectMediaMethod, blockingStub.getCallOptions(), mediaRequest); + blockingStub.getChannel(), readObjectMethod, blockingStub.getCallOptions(), readRequest); } else { - resIterator = blockingStub.getObjectMedia(mediaRequest); + resIterator = blockingStub.readObject(readRequest); } try { while (true) { - GetObjectMediaResponse res = resIterator.next(); - // When zero-copy mashaller is used, the stream that backs GetObjectMediaResponse + ReadObjectResponse res = resIterator.next(); + // When zero-copy mashaller is used, the stream that backs ReadObjectResponse // should be closed when the mssage is no longed needed so that all buffers in the // stream can be reclaimed. If zero-copy is not used, stream will be null. - InputStream stream = getObjectMediaResponseMarshaller.popStream(res); + InputStream stream = ReadObjectResponseMarshaller.popStream(res); try { // Just copy to scratch memory to ensure its data is consumed. ByteString content = res.getChecksummedData().getContent(); @@ -229,16 +236,16 @@ private void makeMediaRequest(ManagedChannel channel, ResultTable results) { } } - private void makeRandomMediaRequest(ManagedChannel channel, ResultTable results) { - StorageGrpc.StorageBlockingStub blockingStub = + private void makeRandomReadRequest(ManagedChannel channel, ResultTable results) { + StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { blockingStub = blockingStub.withCallCredentials( MoreCallCredentials.from(creds)); } - GetObjectMediaRequest.Builder reqBuilder = - GetObjectMediaRequest.newBuilder().setBucket(args.bkt).setObject(args.obj); + ReadObjectRequest.Builder reqBuilder = + ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(args.obj); Random r = new Random(); long buffSize = args.buffSize * 1024; @@ -247,15 +254,15 @@ private void makeRandomMediaRequest(ManagedChannel channel, ResultTable results) long offset = (long) r.nextInt(args.size - args.buffSize) * 1024; reqBuilder.setReadOffset(offset); reqBuilder.setReadLimit(buffSize); - GetObjectMediaRequest req = reqBuilder.build(); + ReadObjectRequest req = reqBuilder.build(); long start = System.currentTimeMillis(); - Iterator resIterator = blockingStub.getObjectMedia(req); + Iterator resIterator = blockingStub.readObject(req); int itr = 0; long bytesRead = 0; while (resIterator.hasNext()) { itr++; - GetObjectMediaResponse res = resIterator.next(); + ReadObjectResponse res = resIterator.next(); bytesRead += res.getChecksummedData().getSerializedSize(); //logger.info("result: " + res.getChecksummedData()); } @@ -282,11 +289,11 @@ private void makeInsertRequest(ManagedChannel channel, ResultTable results, int boolean isLast = false; final CountDownLatch finishLatch = new CountDownLatch(1); - StreamObserver responseObserver = new StreamObserver() { + StreamObserver responseObserver = new StreamObserver() { long start = System.currentTimeMillis(); @Override - public void onNext(Object value) { + public void onNext(WriteObjectResponse value) { } @Override @@ -311,7 +318,7 @@ public void onCompleted() { } }; - StreamObserver requestObserver = asyncStub.insertObject(responseObserver); + StreamObserver requestObserver = asyncStub.writeObject(responseObserver); while (offset < totalBytes) { int add; @@ -324,7 +331,7 @@ public void onCompleted() { isLast = true; } - InsertObjectRequest req = getInsertRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), idx); + WriteObjectRequest req = getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), idx); requestObserver.onNext(req); if (finishLatch.getCount() == 0) { logger.warning("Stream completed before finishing sending requests"); @@ -342,12 +349,12 @@ public void onCompleted() { } - private InsertObjectRequest getInsertRequest(boolean first, boolean last, int offset, ByteString bytes, int idx) { - InsertObjectRequest.Builder builder = InsertObjectRequest.newBuilder(); + private WriteObjectRequest getWriteRequest(boolean first, boolean last, int offset, ByteString bytes, int idx) { + WriteObjectRequest.Builder builder = WriteObjectRequest.newBuilder(); if (first) { - builder.setInsertObjectSpec( - InsertObjectSpec.newBuilder().setResource( - Object.newBuilder().setBucket(args.bkt).setName(args.obj + "_" + idx) + builder.setWriteObjectSpec( + WriteObjectSpec.newBuilder().setResource( + Object.newBuilder().setBucket(toV2BucketName(args.bkt)).setName(args.obj + "_" + idx) ).build() ); } diff --git a/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage.proto b/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage.proto deleted file mode 100644 index 2d0a7a0f..00000000 --- a/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage.proto +++ /dev/null @@ -1,1986 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; - -package google.storage.v1; - -import "google/iam/v1/iam_policy.proto"; -import "google/iam/v1/policy.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/field_mask.proto"; -import "google/protobuf/wrappers.proto"; -import "google/storage/v1/storage_resources.proto"; -import "google/api/client.proto"; - -option go_package = "google.golang.org/genproto/googleapis/storage/v1;storage"; -option java_multiple_files = true; -option java_package = "com.google.google.storage.v1"; - -// Manages Google Cloud Storage resources. -service Storage { - option (google.api.default_host) = "storage.googleapis.com"; - option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform," - "https://www.googleapis.com/auth/cloud-platform.read-only," - "https://www.googleapis.com/auth/devstorage.full_control," - "https://www.googleapis.com/auth/devstorage.read_only," - "https://www.googleapis.com/auth/devstorage.read_write"; - - // Permanently deletes the ACL entry for the specified entity on the specified - // bucket. - rpc DeleteBucketAccessControl(DeleteBucketAccessControlRequest) returns (google.protobuf.Empty) { - } - - // Returns the ACL entry for the specified entity on the specified bucket. - rpc GetBucketAccessControl(GetBucketAccessControlRequest) returns (BucketAccessControl) { - } - - // Creates a new ACL entry on the specified bucket. - rpc InsertBucketAccessControl(InsertBucketAccessControlRequest) returns (BucketAccessControl) { - } - - // Retrieves ACL entries on the specified bucket. - rpc ListBucketAccessControls(ListBucketAccessControlsRequest) returns (ListBucketAccessControlsResponse) { - } - - // Updates an ACL entry on the specified bucket. Equivalent to - // PatchBucketAccessControl, but all unspecified fields will be - // reset to their default values. - rpc UpdateBucketAccessControl(UpdateBucketAccessControlRequest) returns (BucketAccessControl) { - } - - // Updates an ACL entry on the specified bucket. - rpc PatchBucketAccessControl(PatchBucketAccessControlRequest) returns (BucketAccessControl) { - } - - // Permanently deletes an empty bucket. - rpc DeleteBucket(DeleteBucketRequest) returns (google.protobuf.Empty) { - } - - // Returns metadata for the specified bucket. - rpc GetBucket(GetBucketRequest) returns (Bucket) { - } - - // Creates a new bucket. - rpc InsertBucket(InsertBucketRequest) returns (Bucket) { - } - - // Retrieves a list of buckets for a given project. - rpc ListBuckets(ListBucketsRequest) returns (ListBucketsResponse) { - } - - // Locks retention policy on a bucket. - rpc LockBucketRetentionPolicy(LockRetentionPolicyRequest) returns (Bucket) { - } - - // Gets the IAM policy for the specified bucket. - rpc GetBucketIamPolicy(google.iam.v1.GetIamPolicyRequest) returns (google.iam.v1.Policy) { - } - - // Updates an IAM policy for the specified bucket. - rpc SetBucketIamPolicy(google.iam.v1.SetIamPolicyRequest) returns (google.iam.v1.Policy) { - } - - // Tests a set of permissions on the given bucket to see which, if - // any, are held by the caller. - rpc TestBucketIamPermissions(google.iam.v1.TestIamPermissionsRequest) returns (google.iam.v1.TestIamPermissionsResponse) { - } - - // Updates a bucket. Changes to the bucket will be readable immediately after - // writing, but configuration changes may take time to propagate. - rpc PatchBucket(PatchBucketRequest) returns (Bucket) { - } - - // Updates a bucket. Equivalent to PatchBucket, but always replaces all - // mutatable fields of the bucket with new values, reverting all - // unspecified fields to their default values. - // Like PatchBucket, Changes to the bucket will be readable immediately after - // writing, but configuration changes may take time to propagate. - rpc UpdateBucket(UpdateBucketRequest) returns (Bucket) { - } - - // Halts "Object Change Notification" push messagages. - // See https://cloud.google.com/storage/docs/object-change-notification - // Note: this is not related to the newer "Notifications" resource, which - // are stopped using DeleteNotification. - rpc StopChannel(StopChannelRequest) returns (google.protobuf.Empty) { - } - - // Permanently deletes the default object ACL entry for the specified entity - // on the specified bucket. - rpc DeleteDefaultObjectAccessControl(DeleteDefaultObjectAccessControlRequest) returns (google.protobuf.Empty) { - } - - // Returns the default object ACL entry for the specified entity on the - // specified bucket. - rpc GetDefaultObjectAccessControl(GetDefaultObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Creates a new default object ACL entry on the specified bucket. - rpc InsertDefaultObjectAccessControl(InsertDefaultObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Retrieves default object ACL entries on the specified bucket. - rpc ListDefaultObjectAccessControls(ListDefaultObjectAccessControlsRequest) returns (ListObjectAccessControlsResponse) { - } - - // Updates a default object ACL entry on the specified bucket. - rpc PatchDefaultObjectAccessControl(PatchDefaultObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Updates a default object ACL entry on the specified bucket. Equivalent to - // PatchDefaultObjectAccessControl, but modifies all unspecified fields to - // their default values. - rpc UpdateDefaultObjectAccessControl(UpdateDefaultObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Permanently deletes a notification subscription. - // Note: Older, "Object Change Notification" push subscriptions should be - // deleted using StopChannel instead. - rpc DeleteNotification(DeleteNotificationRequest) returns (google.protobuf.Empty) { - } - - // View a notification configuration. - rpc GetNotification(GetNotificationRequest) returns (Notification) { - } - - // Creates a notification subscription for a given bucket. - // These notifications, when triggered, publish messages to the specified - // Cloud Pub/Sub topics. - // See https://cloud.google.com/storage/docs/pubsub-notifications. - rpc InsertNotification(InsertNotificationRequest) returns (Notification) { - } - - // Retrieves a list of notification subscriptions for a given bucket. - rpc ListNotifications(ListNotificationsRequest) returns (ListNotificationsResponse) { - } - - // Permanently deletes the ACL entry for the specified entity on the specified - // object. - rpc DeleteObjectAccessControl(DeleteObjectAccessControlRequest) returns (google.protobuf.Empty) { - } - - // Returns the ACL entry for the specified entity on the specified object. - rpc GetObjectAccessControl(GetObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Creates a new ACL entry on the specified object. - rpc InsertObjectAccessControl(InsertObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Retrieves ACL entries on the specified object. - rpc ListObjectAccessControls(ListObjectAccessControlsRequest) returns (ListObjectAccessControlsResponse) { - } - - // Updates an ACL entry on the specified object. - rpc UpdateObjectAccessControl(UpdateObjectAccessControlRequest) returns (ObjectAccessControl) { - } - - // Concatenates a list of existing objects into a new object in the same - // bucket. - rpc ComposeObject(ComposeObjectRequest) returns (Object) { - } - - // Copies a source object to a destination object. Optionally overrides - // metadata. - rpc CopyObject(CopyObjectRequest) returns (Object) { - } - - // Deletes an object and its metadata. Deletions are permanent if versioning - // is not enabled for the bucket, or if the generation parameter - // is used. - rpc DeleteObject(DeleteObjectRequest) returns (google.protobuf.Empty) { - } - - // Retrieves an object's metadata. - rpc GetObject(GetObjectRequest) returns (Object) { - } - - // Reads an object's data. - rpc GetObjectMedia(GetObjectMediaRequest) returns (stream GetObjectMediaResponse) { - } - - // Stores a new object and metadata. - // - // An object can be written either in a single message stream or in a - // resumable sequence of message streams. To write using a single stream, - // the client should include in the first message of the stream an - // `InsertObjectSpec` describing the destination bucket, object, and any - // preconditions. Additionally, the final message must set 'finish_write' to - // true, or else it is an error. - // - // For a resumable write, the client should instead call - // `StartResumableWrite()` and provide that method an `InsertObjectSpec.` - // They should then attach the returned `upload_id` to the first message of - // each following call to `Insert`. If there is an error or the connection is - // broken during the resumable `Insert()`, the client should check the status - // of the `Insert()` by calling `QueryWriteStatus()` and continue writing from - // the returned `committed_size`. This may be less than the amount of data the - // client previously sent. - // - // The service will not view the object as complete until the client has - // sent an `Insert` with `finish_write` set to `true`. Sending any - // requests on a stream after sending a request with `finish_write` set to - // `true` will cause an error. The client **should** check the - // `Object` it receives to determine how much data the service was - // able to commit and whether the service views the object as complete. - rpc InsertObject(stream InsertObjectRequest) returns (Object) { - } - - // Retrieves a list of objects matching the criteria. - rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse) { - } - - // Rewrites a source object to a destination object. Optionally overrides - // metadata. - rpc RewriteObject(RewriteObjectRequest) returns (RewriteResponse) { - } - - // Starts a resumable write. How long the write operation remains valid, and - // what happens when the write operation becomes invalid, are - // service-dependent. - rpc StartResumableWrite(StartResumableWriteRequest) returns (StartResumableWriteResponse) { - } - - // Determines the `committed_size` for an object that is being written, which - // can then be used as the `write_offset` for the next `Write()` call. - // - // If the object does not exist (i.e., the object has been deleted, or the - // first `Write()` has not yet reached the service), this method returns the - // error `NOT_FOUND`. - // - // The client **may** call `QueryWriteStatus()` at any time to determine how - // much data has been processed for this object. This is useful if the - // client is buffering data and needs to know which data can be safely - // evicted. For any sequence of `QueryWriteStatus()` calls for a given - // object name, the sequence of returned `committed_size` values will be - // non-decreasing. - rpc QueryWriteStatus(QueryWriteStatusRequest) returns (QueryWriteStatusResponse) { - } - - // Updates an object's metadata. - rpc PatchObject(PatchObjectRequest) returns (Object) { - } - - // Updates an object's metadata. Equivalent to PatchObject, but always - // replaces all mutatable fields of the bucket with new values, reverting all - // unspecified fields to their default values. - rpc UpdateObject(UpdateObjectRequest) returns (Object) { - } - - // Gets the IAM policy for the specified object. - rpc GetObjectIamPolicy(google.iam.v1.GetIamPolicyRequest) returns (google.iam.v1.Policy) { - } - - // Updates an IAM policy for the specified object. - rpc SetObjectIamPolicy(google.iam.v1.SetIamPolicyRequest) returns (google.iam.v1.Policy) { - } - - // Tests a set of permissions on the given object to see which, if - // any, are held by the caller. - rpc TestObjectIamPermissions(google.iam.v1.TestIamPermissionsRequest) returns (google.iam.v1.TestIamPermissionsResponse) { - } - - // Watch for changes on all objects in a bucket. - rpc WatchAllObjects(WatchAllObjectsRequest) returns (Channel) { - } - - // Retrieves the name of a project's Google Cloud Storage service account. - rpc GetServiceAccount(GetProjectServiceAccountRequest) returns (ServiceAccount) { - } -} - -// Request message for DeleteBucketAccessControl. -message DeleteBucketAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for GetBucketAccessControl. -message GetBucketAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for InsertBucketAccessControl. -message InsertBucketAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 2; - - // Properties of the new bucket access control being inserted. - BucketAccessControl bucket_access_control = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for ListBucketAccessControl. -message ListBucketAccessControlsRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 2; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 3; -} - -// Request for PatchBucketAccessControl. -message PatchBucketAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 3; - - // The BucketAccessControl for updating. - BucketAccessControl bucket_access_control = 4; - - // List of fields to be updated. - // - // To specify ALL fields, equivalent to the JSON API's "update" function, - // specify a single field with the value `*`. - // - google.protobuf.FieldMask update_mask = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request for UpdateBucketAccessControl. -message UpdateBucketAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 3; - - // The BucketAccessControl for updating. - BucketAccessControl bucket_access_control = 4; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 5; -} - -// Request message for DeleteBucket. -message DeleteBucketRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // If set, only deletes the bucket if its metageneration matches this value. - google.protobuf.Int64Value if_metageneration_match = 2; - - // If set, only deletes the bucket if its metageneration does not match this - // value. - google.protobuf.Int64Value if_metageneration_not_match = 3; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 4; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 5; -} - -// Request message for GetBucket. -message GetBucketRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 2; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 3; - - // Set of properties to return. Defaults to NO_ACL. - CommonEnums.Projection projection = 4; - - // The project to be billed for this request. Required for Requester Pays - // buckets. - string user_project = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request message for InsertBucket. -message InsertBucketRequest { - // Apply a predefined set of access controls to this bucket. - CommonEnums.PredefinedBucketAcl predefined_acl = 1; - - // Apply a predefined set of default object access controls to this bucket. - CommonEnums.PredefinedObjectAcl predefined_default_object_acl = 2; - - // A valid API project identifier. - // Required. - string project = 3; - - // Set of properties to return. Defaults to NO_ACL, unless the - // bucket resource specifies acl or defaultObjectAcl - // properties, when it defaults to FULL. - CommonEnums.Projection projection = 4; - - // The project to be billed for this request. - string user_project = 5; - - // Properties of the new bucket being inserted, including its name. - Bucket bucket = 6; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 7; -} - -// Request message for ListBuckets. -message ListBucketsRequest { - // Maximum number of buckets to return in a single response. The service will - // use this parameter or 1,000 items, whichever is smaller. - int32 max_results = 1; - - // A previously-returned page token representing part of the larger set of - // results to view. - string page_token = 2; - - // Filter results to buckets whose names begin with this prefix. - string prefix = 3; - - // A valid API project identifier. - // Required. - string project = 4; - - // Set of properties to return. Defaults to NO_ACL. - CommonEnums.Projection projection = 5; - - // The project to be billed for this request. - string user_project = 6; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 7; -} - -// Request message for LockRetentionPolicy. -message LockRetentionPolicyRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Makes the operation conditional on whether bucket's current metageneration - // matches the given value. Must be positive. - int64 if_metageneration_match = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request for PatchBucket method. -message PatchBucketRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 2; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 3; - - // Apply a predefined set of access controls to this bucket. - CommonEnums.PredefinedBucketAcl predefined_acl = 4; - - // Apply a predefined set of default object access controls to this bucket. - CommonEnums.PredefinedObjectAcl predefined_default_object_acl = 5; - - // Set of properties to return. Defaults to FULL. - CommonEnums.Projection projection = 6; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 7; - - // The Bucket metadata for updating. - Bucket metadata = 8; - - // List of fields to be updated. - // - // To specify ALL fields, equivalent to the JSON API's "update" function, - // specify a single field with the value `*`. Note: not recommended. If a new - // field is introduced at a later time, an older client updating with the `*` - // may accidentally reset the new field's value. - // - // Not specifying any fields is an error. - // Not specifying a field while setting that field to a non-default value is - // an error. - google.protobuf.FieldMask update_mask = 9; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 10; -} - -// Request for UpdateBucket method. -message UpdateBucketRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 2; - - // Makes the return of the bucket metadata conditional on whether the bucket's - // current metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 3; - - // Apply a predefined set of access controls to this bucket. - CommonEnums.PredefinedBucketAcl predefined_acl = 4; - - // Apply a predefined set of default object access controls to this bucket. - CommonEnums.PredefinedObjectAcl predefined_default_object_acl = 5; - - // Set of properties to return. Defaults to FULL. - CommonEnums.Projection projection = 6; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 7; - - // The Bucket metadata for updating. - Bucket metadata = 8; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 9; -} - -// Request message for StopChannel. -message StopChannelRequest { - // The channel to be stopped. - Channel channel = 1; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 2; -} - -// Request message for DeleteDefaultObjectAccessControl. -message DeleteDefaultObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for GetDefaultObjectAccessControl. -message GetDefaultObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for InsertDefaultObjectAccessControl. -message InsertDefaultObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 2; - - // Properties of the object access control being inserted. - ObjectAccessControl object_access_control = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for ListDefaultObjectAccessControls. -message ListDefaultObjectAccessControlsRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // If present, only return default ACL listing if the bucket's current - // metageneration matches this value. - google.protobuf.Int64Value if_metageneration_match = 2; - - // If present, only return default ACL listing if the bucket's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 3; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 4; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 5; -} - -// Request message for PatchDefaultObjectAccessControl. -message PatchDefaultObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // The ObjectAccessControl for updating. - ObjectAccessControl object_access_control = 4; - - // List of fields to be updated. - // - // To specify ALL fields, equivalent to the JSON API's "update" function, - // specify a single field with the value `*`. Note: not recommended. If a new - // field is introduced at a later time, an older client updating with the `*` - // may accidentally reset the new field's value. - // - // Not specifying any fields is an error. - // Not specifying a field while setting that field to a non-default value is - // an error. - google.protobuf.FieldMask update_mask = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request message for UpdateDefaultObjectAccessControl. -message UpdateDefaultObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // The ObjectAccessControl for updating. - ObjectAccessControl object_access_control = 4; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 5; -} - -// Request message for DeleteNotification. -message DeleteNotificationRequest { - // The parent bucket of the notification. - // Required. - string bucket = 1; - - // ID of the notification to delete. - // Required. - string notification = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for GetNotification. -message GetNotificationRequest { - // The parent bucket of the notification. - // Required. - string bucket = 1; - - // Notification ID. - // Required. - string notification = 2; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for InsertNotification. -message InsertNotificationRequest { - // The parent bucket of the notification. - // Required. - string bucket = 1; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 2; - - // Properties of the notification to be inserted. - Notification notification = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Request message for ListNotifications. -message ListNotificationsRequest { - // Name of a Google Cloud Storage bucket. - // Required. - string bucket = 1; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 2; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 3; -} - -// Request message for DeleteObjectAccessControl. -message DeleteObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // Name of the object. - // Required. - string object = 3; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 4; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request message for GetObjectAccessControl. -message GetObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // Name of the object. - // Required. - string object = 3; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 4; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request message for InsertObjectAccessControl. -message InsertObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Name of the object. - // Required. - string object = 2; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 3; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 4; - - // Properties of the object access control to be inserted. - ObjectAccessControl object_access_control = 5; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 6; -} - -// Request message for ListObjectAccessControls. -message ListObjectAccessControlsRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // Name of the object. - // Required. - string object = 2; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 3; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 4; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 5; -} - -// Request message for UpdateObjetAccessControl. -message UpdateObjectAccessControlRequest { - // Name of a bucket. - // Required. - string bucket = 1; - - // The entity holding the permission. Can be - // user-userId, - // user-emailAddress, - // group-groupId, - // group-emailAddress, allUsers, or - // allAuthenticatedUsers. - // Required. - string entity = 2; - - // Name of the object. - // Required. - string object = 3; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 4; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 5; - - // The ObjectAccessControl for updating. - ObjectAccessControl object_access_control = 6; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 7; - - // List of fields to be updated. - // - // To specify ALL fields, equivalent to the JSON API's "update" function, - // specify a single field with the value `*`. Note: not recommended. If a new - // field is introduced at a later time, an older client updating with the `*` - // may accidentally reset the new field's value. - // - // Not specifying any fields is an error. - // Not specifying a field while setting that field to a non-default value is - // an error. - google.protobuf.FieldMask update_mask = 8; -} - -// Request message for ComposeObject. -message ComposeObjectRequest { - // Description of a source object for a composition request. - message SourceObjects { - // Preconditions for a source object of a composition request. - message ObjectPreconditions { - // Only perform the composition if the generation of the source object - // that would be used matches this value. If this value and a generation - // are both specified, they must be the same value or the call will fail. - google.protobuf.Int64Value if_generation_match = 1; - } - - // The source object's name. All source objects must reside in the same - // bucket. - string name = 1; - - // The generation of this object to use as the source. - int64 generation = 2; - - // Conditions that must be met for this operation to execute. - ObjectPreconditions object_preconditions = 3; - } - - // Name of the bucket containing the source objects. The destination object is - // stored in this bucket. - // Required. - string destination_bucket = 1; - - // Name of the new object. - // Required. - string destination_object = 2; - - // Apply a predefined set of access controls to the destination object. - CommonEnums.PredefinedObjectAcl destination_predefined_acl = 3; - - // Properties of the resulting object. - Object destination = 11; - - // The list of source objects that will be concatenated into a single object. - repeated SourceObjects source_objects = 12; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 4; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 5; - - // Resource name of the Cloud KMS key, of the form - // projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key, - // that will be used to encrypt the object. Overrides the object - // metadata's kms_key_name value, if any. - string kms_key_name = 6; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 7; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 9; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 10; -} - -// Request message for CopyObject. -message CopyObjectRequest { - // Name of the bucket in which to store the new object. Overrides the provided - // object - // metadata's bucket value, if any. - // Required. - string destination_bucket = 1; - - // Name of the new object. - // Required when the object metadata is not otherwise provided. Overrides the - // object metadata's name value, if any. - // Required. - string destination_object = 2; - - // Apply a predefined set of access controls to the destination object. - CommonEnums.PredefinedObjectAcl destination_predefined_acl = 3; - - // Makes the operation conditional on whether the destination object's current - // generation matches the given value. Setting to 0 makes the operation - // succeed only if there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 4; - - // Makes the operation conditional on whether the destination object's current - // generation does not match the given value. If no live object exists, the - // precondition fails. Setting to 0 makes the operation succeed only if there - // is a live version of the object. - google.protobuf.Int64Value if_generation_not_match = 5; - - // Makes the operation conditional on whether the destination object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 6; - - // Makes the operation conditional on whether the destination object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 7; - - // Makes the operation conditional on whether the source object's current - // generation matches the given value. - google.protobuf.Int64Value if_source_generation_match = 8; - - // Makes the operation conditional on whether the source object's current - // generation does not match the given value. - google.protobuf.Int64Value if_source_generation_not_match = 9; - - // Makes the operation conditional on whether the source object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_source_metageneration_match = 10; - - // Makes the operation conditional on whether the source object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_source_metageneration_not_match = 11; - - // Set of properties to return. Defaults to NO_ACL, unless the - // object resource specifies the acl property, when it defaults - // to full. - CommonEnums.Projection projection = 12; - - // Name of the bucket in which to find the source object. - // Required. - string source_bucket = 13; - - // Name of the source object. - // Required. - string source_object = 14; - - // If present, selects a specific revision of the source object (as opposed to - // the latest version, the default). - int64 source_generation = 15; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 16; - - // Properties of the resulting object. If not set, duplicate properties of - // source object. - Object destination = 17; - - // Resource name of the Cloud KMS key, of the form - // projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key, - // that will be used to encrypt the object. Overrides the object - // metadata's kms_key_name value, if any. - string destination_kms_key_name = 20; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 18; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 19; -} - -// Message for deleting an object. -// Either `bucket` and `object` *or* `upload_id` **must** be set (but not both). -message DeleteObjectRequest { - // Name of the bucket in which the object resides. - // Required. - string bucket = 1; - - // The name of the object to delete (when not using a resumable write). - // Required. - string object = 2; - - // The resumable upload_id of the object to delete (when using a - // resumable write). This should be copied from the `upload_id` field of - // `StartResumableWriteResponse`. - string upload_id = 3; - - // If present, permanently deletes a specific revision of this object (as - // opposed to the latest version, the default). - int64 generation = 4; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 5; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 6; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 7; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 8; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 9; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 10; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 11; -} - -// Request message for GetObjectMedia. -message GetObjectMediaRequest { - // The name of the bucket containing the object to read. - string bucket = 1; - - // The name of the object to read. - string object = 2; - - // If present, selects a specific revision of this object (as opposed - // to the latest version, the default). - int64 generation = 3; - - // The offset for the first byte to return in the read, relative to the start - // of the object. - // - // A `read_offset` that is negative or greater than the size of the object - // will cause an `OUT_OF_RANGE` error. - int64 read_offset = 4; - - // The maximum number of `data` bytes the server is allowed to return in the - // sum of all `Object` messages. A `read_limit` of zero indicates that there - // is no limit, and a negative `read_limit` will cause an error. - // - // If the stream returns fewer bytes than allowed by the `read_limit` and no - // error occurred, the stream includes all data from the `read_offset` to the - // end of the resource. - int64 read_limit = 5; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 6; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 7; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 8; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 9; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 10; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 11; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 12; -} - -// Request message for GetObject. -message GetObjectRequest { - // Name of the bucket in which the object resides. - // Required. - string bucket = 1; - - // Name of the object. - // Required. - string object = 2; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 3; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 4; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 5; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 6; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 7; - - // Set of properties to return. Defaults to NO_ACL. - CommonEnums.Projection projection = 8; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 9; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 10; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 11; -} - -// Response message for GetObject. -message GetObjectMediaResponse { - // A portion of the data for the object. The service **may** leave `data` - // empty for any given `ReadResponse`. This enables the service to inform the - // client that the request is still live while it is running an operation to - // generate more data. - ChecksummedData checksummed_data = 1; - - // The checksums of the complete object. The client should compute one of - // these checksums over the downloaded object and compare it against the value - // provided here. - ObjectChecksums object_checksums = 2; - - // If read_offset and or read_limit was specified on the - // GetObjectMediaRequest, ContentRange will be populated on the first - // GetObjectMediaResponse message of the read stream. - ContentRange content_range = 3; -} - -// Describes an attempt to insert an object, possibly over multiple requests. -message InsertObjectSpec { - // Destination object, including its name and its metadata. - Object resource = 1; - - // Apply a predefined set of access controls to this object. - CommonEnums.PredefinedObjectAcl predefined_acl = 2; - - // Makes the operation conditional on whether the object's current - // generation matches the given value. Setting to 0 makes the operation - // succeed only if there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 3; - - // Makes the operation conditional on whether the object's current - // generation does not match the given value. If no live object exists, the - // precondition fails. Setting to 0 makes the operation succeed only if - // there is a live version of the object. - google.protobuf.Int64Value if_generation_not_match = 4; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 5; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 6; - - // Set of properties to return. Defaults to NO_ACL, unless the - // object resource specifies the acl property, when it defaults - // to full. - CommonEnums.Projection projection = 7; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 8; -} - -// Message for writing an object. -message InsertObjectRequest { - // The first message of each stream should set one of the following. - oneof first_message { - // For resumable uploads. This should be the `upload_id` returned from a - // call to `StartResumableWriteResponse`. - string upload_id = 1; - - // For non-resumable uploads. Describes the overall upload, including the - // destination bucket and object name, preconditions, etc. - InsertObjectSpec insert_object_spec = 2; - } - - // The offset from the beginning of the object at which the data should be - // written. - // Required. - // - // In the first `InsertObjectRequest` of a `InsertObject()` action, it - // indicates the initial offset for the `Insert()` call. The value **must** be - // equal to the `committed_size` that a call to `QueryWriteStatus()` would - // return (0 if this is the first write to the object). - // - // On subsequent calls, this value **must** be no larger than the sum of the - // first `write_offset` and the sizes of all `data` chunks sent previously on - // this stream. - // - // An incorrect value will cause an error. - int64 write_offset = 3; - - // A portion of the data for the object. - oneof data { - // The data to insert. If a crc32c checksum is provided that doesn't match - // the checksum computed by the service, the request will fail. - ChecksummedData checksummed_data = 4; - - // A reference to an existing object. This can be used to support - // several use cases: - // - Writing a sequence of data buffers supports the basic use case of - // uploading a complete object, chunk by chunk. - // - Writing a sequence of references to existing objects allows an - // object to be composed from a collection of objects, which can be - // used to support parallel object writes. - // - Writing a single reference with a given offset and size can be used - // to create an object from a slice of an existing object. - // - Writing an object referencing a object slice (created as noted - // above) followed by a data buffer followed by another object - // slice can be used to support delta upload functionality. - GetObjectMediaRequest reference = 5; - } - - // Checksums for the complete object. If the checksums computed by the service - // don't match the specifified checksums the call will fail. May only be - // provided in the first or last request (either with first_message, or - // finish_write set). - ObjectChecksums object_checksums = 6; - - // If `true`, this indicates that the write is complete. Sending any - // `InsertObjectRequest`s subsequent to one in which `finish_write` is `true` - // will cause an error. - // For a non-resumable write (where the upload_id was not set in the first - // message), it is an error not to set this field in the final message of the - // stream. - bool finish_write = 7; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 8; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 9; -} - -// Request message for ListObjects. -message ListObjectsRequest { - // Name of the bucket in which to look for objects. - // Required. - string bucket = 1; - - // Returns results in a directory-like mode. items will contain - // only objects whose names, aside from the prefix, do not - // contain delimiter. Objects whose names, aside from the - // prefix, contain delimiter will have their name, - // truncated after the delimiter, returned in - // prefixes. Duplicate prefixes are omitted. - string delimiter = 2; - - // If true, objects that end in exactly one instance of delimiter - // will have their metadata included in items in addition to - // prefixes. - bool include_trailing_delimiter = 3; - - // Maximum number of items plus prefixes to return - // in a single page of responses. As duplicate prefixes are - // omitted, fewer total results may be returned than requested. The service - // will use this parameter or 1,000 items, whichever is smaller. - int32 max_results = 4; - - // A previously-returned page token representing part of the larger set of - // results to view. - string page_token = 5; - - // Filter results to objects whose names begin with this prefix. - string prefix = 6; - - // Set of properties to return. Defaults to NO_ACL. - CommonEnums.Projection projection = 7; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 8; - - // If true, lists all versions of an object as distinct results. - // The default is false. For more information, see Object Versioning. - bool versions = 9; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 10; -} - -// Request object for `ByteStream.QueryWriteStatus`. -message QueryWriteStatusRequest { - // The name of the resume token for the object whose write status is being - // requested. - // Required. - string upload_id = 1; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 2; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 3; -} - -// Response object for `ByteStream.QueryWriteStatus`. -message QueryWriteStatusResponse { - // The number of bytes that have been processed for the given object. - int64 committed_size = 1; - - // `complete` is `true` only if the client has sent a `InsertObjectRequest` - // with `finish_write` set to true, and the server has processed that request. - bool complete = 2; -} - -// Request message for RewriteObject. -message RewriteObjectRequest { - // Name of the bucket in which to store the new object. Overrides the provided - // object metadata's bucket value, if any. - // Required. - string destination_bucket = 1; - - // Name of the new object. - // Required when the object metadata is not otherwise provided. Overrides the - // object metadata's name value, if any. - // Required. - string destination_object = 2; - - // Resource name of the Cloud KMS key, of the form - // projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key, - // that will be used to encrypt the object. Overrides the object - // metadata's kms_key_name value, if any. - string destination_kms_key_name = 3; - - // Apply a predefined set of access controls to the destination object. - CommonEnums.PredefinedObjectAcl destination_predefined_acl = 4; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 5; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 6; - - // Makes the operation conditional on whether the destination object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 7; - - // Makes the operation conditional on whether the destination object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 8; - - // Makes the operation conditional on whether the source object's current - // generation matches the given value. - google.protobuf.Int64Value if_source_generation_match = 9; - - // Makes the operation conditional on whether the source object's current - // generation does not match the given value. - google.protobuf.Int64Value if_source_generation_not_match = 10; - - // Makes the operation conditional on whether the source object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_source_metageneration_match = 11; - - // Makes the operation conditional on whether the source object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_source_metageneration_not_match = 12; - - // The maximum number of bytes that will be rewritten per rewrite request. - // Most callers - // shouldn't need to specify this parameter - it is primarily in place to - // support testing. If specified the value must be an integral multiple of - // 1 MiB (1048576). Also, this only applies to requests where the source and - // destination span locations and/or storage classes. Finally, this value must - // not change across rewrite calls else you'll get an error that the - // rewriteToken is invalid. - int64 max_bytes_rewritten_per_call = 13; - - // Set of properties to return. Defaults to NO_ACL, unless the - // object resource specifies the acl property, when it defaults - // to full. - CommonEnums.Projection projection = 14; - - // Include this field (from the previous rewrite response) on each rewrite - // request after the first one, until the rewrite response 'done' flag is - // true. Calls that provide a rewriteToken can omit all other request fields, - // but if included those fields must match the values provided in the first - // rewrite request. - string rewrite_token = 15; - - // Name of the bucket in which to find the source object. - // Required. - string source_bucket = 16; - - // Name of the source object. - // Required. - string source_object = 17; - - // If present, selects a specific revision of the source object (as opposed to - // the latest version, the default). - int64 source_generation = 18; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 19; - - // Properties of the destination, post-rewrite object. - Object object = 20; - - // The algorithm used to encrypt the source object, if any. - string copy_source_encryption_algorithm = 21; - - // The encryption key used to encrypt the source object, if any. - string copy_source_encryption_key = 22; - - // The SHA-256 hash of the key used to encrypt the source object, if any. - string copy_source_encryption_key_sha256 = 23; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 24; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 25; -} - -// A rewrite response. -message RewriteResponse { - // The total bytes written so far, which can be used to provide a waiting user - // with a progress indicator. This property is always present in the response. - int64 total_bytes_rewritten = 1; - - // The total size of the object being copied in bytes. This property is always - // present in the response. - int64 object_size = 2; - - // true if the copy is finished; otherwise, false if - // the copy is in progress. This property is always present in the response. - bool done = 3; - - // A token to use in subsequent requests to continue copying data. This token - // is present in the response only when there is more data to copy. - string rewrite_token = 4; - - // A resource containing the metadata for the copied-to object. This property - // is present in the response only when copying completes. - Object resource = 5; -} - -// Request message StartResumableWrite. -message StartResumableWriteRequest { - // The destination bucket, object, and metadata, as well as any preconditions. - InsertObjectSpec insert_object_spec = 1; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 3; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 4; -} - -// Response object for ByteStream.StartResumableWrite. -message StartResumableWriteResponse { - // The upload_id of the newly started resumable write operation. This - // value should be copied into the `InsertObjectRequest.upload_id` field. - string upload_id = 1; -} - -// Request message for PatchObject. -message PatchObjectRequest { - // Name of the bucket in which the object resides. - // Required. - string bucket = 1; - - // Name of the object. - // Required. - string object = 2; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 3; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 4; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 5; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 6; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 7; - - // Apply a predefined set of access controls to this object. - CommonEnums.PredefinedObjectAcl predefined_acl = 8; - - // Set of properties to return. Defaults to FULL. - CommonEnums.Projection projection = 9; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 10; - - // The Object metadata for updating. - Object metadata = 11; - - // List of fields to be updated. - // - // To specify ALL fields, equivalent to the JSON API's "update" function, - // specify a single field with the value `*`. Note: not recommended. If a new - // field is introduced at a later time, an older client updating with the `*` - // may accidentally reset the new field's value. - // - // Not specifying any fields is an error. - // Not specifying a field while setting that field to a non-default value is - // an error. - google.protobuf.FieldMask update_mask = 12; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 13; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 14; -} - -// Request message for UpdateObject. -message UpdateObjectRequest { - // Name of the bucket in which the object resides. - // Required. - string bucket = 1; - - // Name of the object. - // Required. - string object = 2; - - // If present, selects a specific revision of this object (as opposed to the - // latest version, the default). - int64 generation = 3; - - // Makes the operation conditional on whether the object's current generation - // matches the given value. Setting to 0 makes the operation succeed only if - // there are no live versions of the object. - google.protobuf.Int64Value if_generation_match = 4; - - // Makes the operation conditional on whether the object's current generation - // does not match the given value. If no live object exists, the precondition - // fails. Setting to 0 makes the operation succeed only if there is a live - // version of the object. - google.protobuf.Int64Value if_generation_not_match = 5; - - // Makes the operation conditional on whether the object's current - // metageneration matches the given value. - google.protobuf.Int64Value if_metageneration_match = 6; - - // Makes the operation conditional on whether the object's current - // metageneration does not match the given value. - google.protobuf.Int64Value if_metageneration_not_match = 7; - - // Apply a predefined set of access controls to this object. - CommonEnums.PredefinedObjectAcl predefined_acl = 8; - - // Set of properties to return. Defaults to FULL. - CommonEnums.Projection projection = 9; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 10; - - // The Object metadata for updating. - Object metadata = 11; - - // A set of parameters common to Storage API requests concerning an object. - CommonObjectRequestParams common_object_request_params = 12; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 13; -} - -// Request message for WatchAllObjects. -message WatchAllObjectsRequest { - // Name of the bucket in which to look for objects. - string bucket = 1; - - // If true, lists all versions of an object as distinct results. - // The default is false. For more information, see Object Versioning. - bool versions = 2; - - // Returns results in a directory-like mode. items will contain - // only objects whose names, aside from the prefix, do not - // contain delimiter. Objects whose names, aside from the - // prefix, contain delimiter will have their name, - // truncated after the delimiter, returned in - // prefixes. Duplicate prefixes are omitted. - string delimiter = 3; - - // Maximum number of items plus prefixes to return - // in a single page of responses. As duplicate prefixes are - // omitted, fewer total results may be returned than requested. The service - // will use this parameter or 1,000 items, whichever is smaller. - int32 max_results = 4; - - // Filter results to objects whose names begin with this prefix. - string prefix = 5; - - // If true, objects that end in exactly one instance of delimiter - // will have their metadata included in items in addition to - // prefixes. - bool include_trailing_delimiter = 6; - - // A previously-returned page token representing part of the larger set of - // results to view. - string page_token = 7; - - // Set of properties to return. Defaults to NO_ACL. - CommonEnums.Projection projection = 8; - - // The project to be billed for this request. - // Required for Requester Pays buckets. - string user_project = 9; - - // Properties of the channel to be inserted. - Channel channel = 10; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 11; -} - -// Request message for GetProjectServiceAccount. -message GetProjectServiceAccountRequest { - // Project ID. - // Required. - string project_id = 1; - - // The project to be billed for this request. - string user_project = 2; - - // A set of parameters common to all Storage API requests. - CommonRequestParams common_request_params = 3; -} - -// Parameters that can be passed to any object request. -message CommonObjectRequestParams { - // Encryption algorithm used with Customer-Supplied Encryption Keys feature. - string encryption_algorithm = 1; - - // Encryption key used with Customer-Supplied Encryption Keys feature. - string encryption_key = 2; - - // SHA256 hash of encryption key used with Customer-Supplied Encryption Keys - // feature. - string encryption_key_sha256 = 3; -} - -// Parameters that can be passed to any request. -message CommonRequestParams { - // Required when using buckets with Requestor Pays feature enabled. - string user_project = 1; - - // Lets you enforce per-user quotas from a server-side application even in - // cases when the user's IP address is unknown. This can occur, for example, - // with applications that run cron jobs on App Engine on a user's behalf. - // You can choose any arbitrary string that uniquely identifies a user, but it - // is limited to 40 characters. - // Overrides user_ip if both are provided. - string quota_user = 2; - - // IP address of the end user for whom the API call is being made. - // Lets you enforce per-user quotas when calling the API from a server-side - // application. - string user_ip = 3; - - // Subset of fields to include in the response. - google.protobuf.FieldMask fields = 4; -} - -// Shared constants. -message ServiceConstants { - // A collection of constant values meaningful to the Storage API. - enum Values { - option allow_alias = true; - - // Unused. Proto3 requires first enum to be 0. - SIZE_UNSPECIFIED = 0; - - // The maximum size chunk that can will be returned in a single - // ReadRequest. - // 2 MiB. - MAX_READ_CHUNK_BYTES = 2097152; - - // The maximum size chunk that can be sent in a single InsertObjectRequest. - // 2 MiB. - MAX_WRITE_CHUNK_BYTES = 2097152; - - // The maximum size of an object in MB - whether written in a single stream - // or composed from multiple other objects. - // 5 TiB. - MAX_OBJECT_SIZE_MB = 5242880; - - // The maximum length field name that can be sent in a single - // custom metadata field. - // 1 KiB. - MAX_CUSTOM_METADATA_FIELD_NAME_BYTES = 1024; - - // The maximum length field value that can be sent in a single - // custom_metadata field. - // 4 KiB. - MAX_CUSTOM_METADATA_FIELD_VALUE_BYTES = 4096; - - // The maximum total bytes that can be populated into all field names and - // values of the custom_metadata for one object. - // 8 KiB. - MAX_CUSTOM_METADATA_TOTAL_SIZE_BYTES = 8192; - - // The maximum total bytes that can be populated into all bucket metadata - // fields. - // 20 KiB. - MAX_BUCKET_METADATA_TOTAL_SIZE_BYTES = 20480; - - // The maximum number of NotificationConfigurations that can be registered - // for a given bucket. - MAX_NOTIFICATION_CONFIGS_PER_BUCKET = 100; - - // The maximum number of LifecycleRules that can be registered for a given - // bucket. - MAX_LIFECYCLE_RULES_PER_BUCKET = 100; - - // The maximum number of custom attributes per NotificationConfig. - MAX_NOTIFICATION_CUSTOM_ATTRIBUTES = 5; - - // The maximum length of a custom attribute key included in - // NotificationConfig. - MAX_NOTIFICATION_CUSTOM_ATTRIBUTE_KEY_LENGTH = 256; - - // The maximum length of a custom attribute value included in a - // NotificationConfig. - MAX_NOTIFICATION_CUSTOM_ATTRIBUTE_VALUE_LENGTH = 1024; - - // The maximum number of key/value entries per bucket label. - MAX_LABELS_ENTRIES_COUNT = 64; - - // The maximum character length of the key or value in a bucket - // label map. - MAX_LABELS_KEY_VALUE_LENGTH = 63; - - // The maximum byte size of the key or value in a bucket label - // map. - MAX_LABELS_KEY_VALUE_BYTES = 128; - - // The maximum number of object IDs that can be included in a - // DeleteObjectsRequest. - MAX_OBJECT_IDS_PER_DELETE_OBJECTS_REQUEST = 1000; - - // The maximum number of days for which a token returned by the - // GetListObjectsSplitPoints RPC is valid. - SPLIT_TOKEN_MAX_VALID_DAYS = 14; - } - - -} diff --git a/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage_resources.proto b/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage_resources.proto deleted file mode 100644 index 77a01785..00000000 --- a/end2end-test-examples/gcs/src/main/proto/google/storage/v1/storage_resources.proto +++ /dev/null @@ -1,793 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; - -package google.storage.v1; - -import "google/protobuf/timestamp.proto"; -import "google/protobuf/wrappers.proto"; - -option go_package = "google.golang.org/genproto/googleapis/storage/v1;storage"; -option java_multiple_files = true; -option java_outer_classname = "CloudStorageResourcesProto"; -option java_package = "com.google.google.storage.v1"; - -// A bucket. -message Bucket { - // Billing properties of a bucket. - message Billing { - // When set to true, Requester Pays is enabled for this bucket. - bool requester_pays = 1; - } - - // Cross-Origin Response sharing (CORS) properties for a bucket. - // For more on GCS and CORS, see - // https://cloud.google.com/storage/docs/cross-origin. - // For more on CORS in general, see https://tools.ietf.org/html/rfc6454. - message Cors { - // The list of Origins eligible to receive CORS response headers. See - // [https://tools.ietf.org/html/rfc6454][RFC 6454] for more on origins. - // Note: "*" is permitted in the list of origins, and means "any Origin". - repeated string origin = 1; - - // The list of HTTP methods on which to include CORS response headers, - // (`GET`, `OPTIONS`, `POST`, etc) Note: "*" is permitted in the list of - // methods, and means "any method". - repeated string method = 2; - - // The list of HTTP headers other than the - // [https://www.w3.org/TR/cors/#simple-response-header][simple response - // headers] to give permission for the user-agent to share across domains. - repeated string response_header = 3; - - // The value, in seconds, to return in the - // [https://www.w3.org/TR/cors/#access-control-max-age-response-header][Access-Control-Max-Age - // header] used in preflight responses. - int32 max_age_seconds = 4; - } - - // Encryption properties of a bucket. - message Encryption { - // A Cloud KMS key that will be used to encrypt objects inserted into this - // bucket, if no encryption method is specified. - string default_kms_key_name = 1; - } - - // Lifecycle properties of a bucket. - // For more information, see https://cloud.google.com/storage/docs/lifecycle. - message Lifecycle { - // A lifecycle Rule, combining an action to take on an object and a - // condition which will trigger that action. - message Rule { - // An action to take on an object. - message Action { - // Type of the action. Currently, only `Delete` and - // `SetStorageClass` are supported. - string type = 1; - - // Target storage class. Required iff the type of the action is - // SetStorageClass. - string storage_class = 2; - } - - // A condition of an object which triggers some action. - message Condition { - // Age of an object (in days). This condition is satisfied when an - // object reaches the specified age. - int32 age = 1; - - // A date in [RFC 3339][1] format with only the date part (for - // instance, "2013-01-15"). This condition is satisfied when an - // object is created before midnight of the specified date in UTC. - // [1]: https://tools.ietf.org/html/rfc3339 - google.protobuf.Timestamp created_before = 2; - - // Relevant only for versioned objects. If the value is - // `true`, this condition matches live objects; if the value - // is `false`, it matches archived objects. - google.protobuf.BoolValue is_live = 3; - - // Relevant only for versioned objects. If the value is N, this - // condition is satisfied when there are at least N versions (including - // the live version) newer than this version of the object. - int32 num_newer_versions = 4; - - // Objects having any of the storage classes specified by this condition - // will be matched. Values include `MULTI_REGIONAL`, `REGIONAL`, - // `NEARLINE`, `COLDLINE`, `STANDARD`, and - // `DURABLE_REDUCED_AVAILABILITY`. - repeated string matches_storage_class = 5; - - // A regular expression that satisfies the RE2 syntax. This condition is - // satisfied when the name of the object matches the RE2 pattern. Note: - // This feature is currently in the "Early Access" launch stage and is - // only available to a whitelisted set of users; that means that this - // feature may be changed in backward-incompatible ways and that it is - // not guaranteed to be released. - string matches_pattern = 6; - } - - // The action to take. - Action action = 1; - - // The condition(s) under which the action will be taken. - Condition condition = 2; - } - - // A lifecycle management rule, which is made of an action to take and the - // condition(s) under which the action will be taken. - repeated Rule rule = 1; - } - - // Logging-related properties of a bucket. - message Logging { - // The destination bucket where the current bucket's logs should be placed. - string log_bucket = 1; - - // A prefix for log object names. - string log_object_prefix = 2; - } - - // Retention policy properties of a bucket. - message RetentionPolicy { - // Server-determined value that indicates the time from which policy was - // enforced and effective. This value is in - // [https://tools.ietf.org/html/rfc3339][RFC 3339] format. - google.protobuf.Timestamp effective_time = 1; - - // Once locked, an object retention policy cannot be modified. - bool is_locked = 2; - - // The duration in seconds that objects need to be retained. Retention - // duration must be greater than zero and less than 100 years. Note that - // enforcement of retention periods less than a day is not guaranteed. Such - // periods should only be used for testing purposes. - int64 retention_period = 3; - } - - // Properties of a bucket related to versioning. - // For more on GCS versioning, see - // https://cloud.google.com/storage/docs/object-versioning. - message Versioning { - // While set to true, versioning is fully enabled for this bucket. - bool enabled = 1; - } - - // Properties of a bucket related to accessing the contents as a static - // website. For more on hosting a static website via GCS, see - // https://cloud.google.com/storage/docs/hosting-static-website. - message Website { - // If the requested object path is missing, the service will ensure the path - // has a trailing '/', append this suffix, and attempt to retrieve the - // resulting object. This allows the creation of `index.html` - // objects to represent directory pages. - string main_page_suffix = 1; - - // If the requested object path is missing, and any - // `mainPageSuffix` object is missing, if applicable, the service - // will return the named object from this bucket as the content for a - // [https://tools.ietf.org/html/rfc7231#section-6.5.4][404 Not Found] - // result. - string not_found_page = 2; - } - - // Access controls on the bucket. - repeated BucketAccessControl acl = 1; - - // Default access controls to apply to new objects when no ACL is provided. - repeated ObjectAccessControl default_object_acl = 2; - - // The bucket's lifecycle configuration. See - // [https://developers.google.com/storage/docs/lifecycle]Lifecycle Management] - // for more information. - Lifecycle lifecycle = 3; - - // The creation time of the bucket in - // [https://tools.ietf.org/html/rfc3339][RFC 3339] format. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp time_created = 4; - - // The ID of the bucket. For buckets, the `id` and `name` properties are the - // same. - // Attempting to update this field after the bucket is created will result in - // an error. - string id = 5; - - // The name of the bucket. - // Attempting to update this field after the bucket is created will result in - // an error. - string name = 6; - - // The project number of the project the bucket belongs to. - // Attempting to set this field will result in an error. - int64 project_number = 7; - - // The metadata generation of this bucket. - // Attempting to set this field will result in an error. - int64 metageneration = 8; - - // The bucket's [https://www.w3.org/TR/cors/][Cross-Origin Resource Sharing] - // (CORS) configuration. - repeated Cors cors = 9; - - // The location of the bucket. Object data for objects in the bucket resides - // in physical storage within this region. Defaults to `US`. See the - // [https://developers.google.com/storage/docs/concepts-techniques#specifyinglocations"][developer's - // guide] for the authoritative list. Attempting to update this field after - // the bucket is created will result in an error. - string location = 10; - - // The bucket's default storage class, used whenever no storageClass is - // specified for a newly-created object. This defines how objects in the - // bucket are stored and determines the SLA and the cost of storage. - // If this value is not specified when the bucket is created, it will default - // to `STANDARD`. For more information, see - // https://developers.google.com/storage/docs/storage-classes. - string storage_class = 11; - - // HTTP 1.1 [https://tools.ietf.org/html/rfc7232#section-2.3"]Entity tag] - // for the bucket. - // Attempting to set this field will result in an error. - string etag = 12; - - // The modification time of the bucket. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp updated = 13; - - // The default value for event-based hold on newly created objects in this - // bucket. Event-based hold is a way to retain objects indefinitely until an - // event occurs, signified by the - // hold's release. After being released, such objects will be subject to - // bucket-level retention (if any). One sample use case of this flag is for - // banks to hold loan documents for at least 3 years after loan is paid in - // full. Here, bucket-level retention is 3 years and the event is loan being - // paid in full. In this example, these objects will be held intact for any - // number of years until the event has occurred (event-based hold on the - // object is released) and then 3 more years after that. That means retention - // duration of the objects begins from the moment event-based hold - // transitioned from true to false. Objects under event-based hold cannot be - // deleted, overwritten or archived until the hold is removed. - bool default_event_based_hold = 14; - - // User-provided labels, in key/value pairs. - map labels = 15; - - // The bucket's website configuration, controlling how the service behaves - // when accessing bucket contents as a web site. See the - // [https://cloud.google.com/storage/docs/static-website][Static Website - // Examples] for more information. - Website website = 16; - - // The bucket's versioning configuration. - Versioning versioning = 17; - - // The bucket's logging configuration, which defines the destination bucket - // and optional name prefix for the current bucket's logs. - Logging logging = 18; - - // The owner of the bucket. This is always the project team's owner group. - Owner owner = 19; - - // Encryption configuration for a bucket. - Encryption encryption = 20; - - // The bucket's billing configuration. - Billing billing = 21; - - // The bucket's retention policy. The retention policy enforces a minimum - // retention time for all objects contained in the bucket, based on their - // creation time. Any attempt to overwrite or delete objects younger than the - // retention period will result in a PERMISSION_DENIED error. An unlocked - // retention policy can be modified or removed from the bucket via a - // storage.buckets.update operation. A locked retention policy cannot be - // removed or shortened in duration for the lifetime of the bucket. - // Attempting to remove or decrease period of a locked retention policy will - // result in a PERMISSION_DENIED error. - RetentionPolicy retention_policy = 22; -} - -// An access-control entry. -message BucketAccessControl { - // The access permission for the entity. - string role = 1; - - // HTTP 1.1 ["https://tools.ietf.org/html/rfc7232#section-2.3][Entity tag] - // for the access-control entry. - string etag = 2; - - // The ID of the access-control entry. - string id = 3; - - // The name of the bucket. - string bucket = 4; - - // The entity holding the permission, in one of the following forms: - // * `user-{userid}` - // * `user-{email}` - // * `group-{groupid}` - // * `group-{email}` - // * `domain-{domain}` - // * `project-{team-projectid}` - // * `allUsers` - // * `allAuthenticatedUsers` - // Examples: - // * The user `liz@example.com` would be `user-liz@example.com`. - // * The group `example@googlegroups.com` would be - // `group-example@googlegroups.com` - // * All members of the Google Apps for Business domain `example.com` would be - // `domain-example.com` - string entity = 6; - - // The ID for the entity, if any. - string entity_id = 7; - - // The email address associated with the entity, if any. - string email = 8; - - // The domain associated with the entity, if any. - string domain = 9; - - // The project team associated with the entity, if any. - ProjectTeam project_team = 10; -} - -// The response to a call to BucketAccessControls.ListBucketAccessControls. -message ListBucketAccessControlsResponse { - // The list of items. - repeated BucketAccessControl items = 1; -} - -// The result of a call to Buckets.ListBuckets -message ListBucketsResponse { - // The list of items. - repeated Bucket items = 1; - - // The continuation token, used to page through large result sets. Provide - // this value in a subsequent request to return the next page of results. - string next_page_token = 2; -} - -// An notification channel used to watch for resource changes. -message Channel { - // A UUID or similar unique string that identifies this channel. - string id = 1; - - // An opaque ID that identifies the resource being watched on this channel. - // Stable across different API versions. - string resource_id = 2; - - // A version-specific identifier for the watched resource. - string resource_uri = 3; - - // An arbitrary string delivered to the target address with each notification - // delivered over this channel. Optional. - string token = 4; - - // Date and time of notification channel expiration. Optional. - google.protobuf.Timestamp expiration = 5; - - // The type of delivery mechanism used for this channel. - string type = 6; - - // The address where notifications are delivered for this channel. - string address = 7; - - // Additional parameters controlling delivery channel behavior. Optional. - map params = 8; - - // A Boolean value to indicate whether payload is wanted. Optional. - bool payload = 9; -} - -// Message used to convey content being read or written, along with its -// checksum. -message ChecksummedData { - // The data. - bytes content = 1; - - // CRC32C digest of the contents. - google.protobuf.UInt32Value crc32c = 2; -} - -// Message used for storing full (not subrange) object checksums. -message ObjectChecksums { - // CRC32C digest of the object data. Computed by the GCS service for - // all written objects, and validated by the GCS service against - // client-supplied values if present in an InsertObjectRequest. - google.protobuf.UInt32Value crc32c = 1; - - // Hex-encoded MD5 hash of the object data (hexdigest). Whether/how this - // checksum is provided and validated is service-dependent. - string md5_hash = 2; -} - -// A collection of enums used in multiple places throughout the API. -message CommonEnums { - // A set of properties to return in a response. - enum Projection { - // No specified projection. - PROJECTION_UNSPECIFIED = 0; - - // Omit `owner`, `acl`, and `defaultObjectAcl` properties. - NO_ACL = 1; - - // Include all properties. - FULL = 2; - } - - // Predefined or "canned" aliases for sets of specific bucket ACL entries. - enum PredefinedBucketAcl { - // No predefined ACL. - PREDEFINED_BUCKET_ACL_UNSPECIFIED = 0; - - // Project team owners get `OWNER` access, and - // `allAuthenticatedUsers` get `READER` access. - BUCKET_ACL_AUTHENTICATED_READ = 1; - - // Project team owners get `OWNER` access. - BUCKET_ACL_PRIVATE = 2; - - // Project team members get access according to their roles. - BUCKET_ACL_PROJECT_PRIVATE = 3; - - // Project team owners get `OWNER` access, and - // `allUsers` get `READER` access. - BUCKET_ACL_PUBLIC_READ = 4; - - // Project team owners get `OWNER` access, and - // `allUsers` get `WRITER` access. - BUCKET_ACL_PUBLIC_READ_WRITE = 5; - } - - // Predefined or "canned" aliases for sets of specific object ACL entries. - enum PredefinedObjectAcl { - // No predefined ACL. - PREDEFINED_OBJECT_ACL_UNSPECIFIED = 0; - - // Object owner gets `OWNER` access, and - // `allAuthenticatedUsers` get `READER` access. - OBJECT_ACL_AUTHENTICATED_READ = 1; - - // Object owner gets `OWNER` access, and project team owners get - // `OWNER` access. - OBJECT_ACL_BUCKET_OWNER_FULL_CONTROL = 2; - - // Object owner gets `OWNER` access, and project team owners get - // `READER` access. - OBJECT_ACL_BUCKET_OWNER_READ = 3; - - // Object owner gets `OWNER` access. - OBJECT_ACL_PRIVATE = 4; - - // Object owner gets `OWNER` access, and project team members get - // access according to their roles. - OBJECT_ACL_PROJECT_PRIVATE = 5; - - // Object owner gets `OWNER` access, and `allUsers` - // get `READER` access. - OBJECT_ACL_PUBLIC_READ = 6; - } - - -} - -// Specifies a requested range of bytes to download. -message ContentRange { - // The starting offset of the object data. - int64 start = 1; - - // The ending offset of the object data. - int64 end = 2; - - // The complete length of the object data. - int64 complete_length = 3; -} - -// A subscription to receive Google PubSub notifications. -message Notification { - // The Cloud PubSub topic to which this subscription publishes. Formatted as: - // '//pubsub.googleapis.com/projects/{project-identifier}/topics/{my-topic}' - string topic = 1; - - // If present, only send notifications about listed event types. If empty, - // sent notifications for all event types. - repeated string event_types = 2; - - // An optional list of additional attributes to attach to each Cloud PubSub - // message published for this notification subscription. - map custom_attributes = 3; - - // HTTP 1.1 [https://tools.ietf.org/html/rfc7232#section-2.3][Entity tag] - // for this subscription notification. - string etag = 4; - - // If present, only apply this notification configuration to object names that - // begin with this prefix. - string object_name_prefix = 5; - - // The desired content of the Payload. - string payload_format = 6; - - // The ID of the notification. - string id = 7; -} - -// The result of a call to Notifications.ListNotifications -message ListNotificationsResponse { - // The list of items. - repeated Notification items = 1; -} - -// An object. -message Object { - // Describes the customer-specified mechanism used to store the data at rest. - message CustomerEncryption { - // The encryption algorithm. - string encryption_algorithm = 1; - - // SHA256 hash value of the encryption key. - string key_sha256 = 2; - } - - // Content-Encoding of the object data, matching - // [https://tools.ietf.org/html/rfc7231#section-3.1.2.2][RFC 7231 §3.1.2.2] - string content_encoding = 1; - - // Content-Disposition of the object data, matching - // [https://tools.ietf.org/html/rfc6266][RFC 6266]. - string content_disposition = 2; - - // Cache-Control directive for the object data, matching - // [https://tools.ietf.org/html/rfc7234#section-5.2"][RFC 7234 §5.2]. - // If omitted, and the object is accessible to all anonymous users, the - // default will be `public, max-age=3600`. - string cache_control = 3; - - // Access controls on the object. - repeated ObjectAccessControl acl = 4; - - // Content-Language of the object data, matching - // [https://tools.ietf.org/html/rfc7231#section-3.1.3.2][RFC 7231 §3.1.3.2]. - string content_language = 5; - - // The version of the metadata for this object at this generation. Used for - // preconditions and for detecting changes in metadata. A metageneration - // number is only meaningful in the context of a particular generation of a - // particular object. - // Attempting to set this field will result in an error. - int64 metageneration = 6; - - // The deletion time of the object. Will be returned if and only if this - // version of the object has been deleted. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp time_deleted = 7; - - // Content-Type of the object data, matching - // [https://tools.ietf.org/html/rfc7231#section-3.1.1.5][RFC 7231 §3.1.1.5]. - // If an object is stored without a Content-Type, it is served as - // `application/octet-stream`. - string content_type = 8; - - // Content-Length of the object data in bytes, matching - // [https://tools.ietf.org/html/rfc7230#section-3.3.2][RFC 7230 §3.3.2]. - // Attempting to set this field will result in an error. - int64 size = 9; - - // The creation time of the object. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp time_created = 10; - - // CRC32c checksum. For more information about using the CRC32c - // checksum, see - // [https://cloud.google.com/storage/docs/hashes-etags#_JSONAPI][Hashes and - // ETags: Best Practices]. This is a server determined value and should not be - // supplied by the user when sending an Object. The server will ignore any - // value provided. Users should instead use the object_checksums field on the - // InsertObjectRequest when uploading an object. - google.protobuf.UInt32Value crc32c = 11; - - // Number of underlying components that make up this object. Components are - // accumulated by compose operations. - // Attempting to set this field will result in an error. - int32 component_count = 12; - - // MD5 hash of the data; encoded using base64 as per - // [https://tools.ietf.org/html/rfc4648#section-4][RFC 4648 §4]. For more - // information about using the MD5 hash, see - // [https://cloud.google.com/storage/docs/hashes-etags#_JSONAPI][Hashes and - // ETags: Best Practices]. This is a server determined value and should not be - // supplied by the user when sending an Object. The server will ignore any - // value provided. Users should instead use the object_checksums field on the - // InsertObjectRequest when uploading an object. - string md5_hash = 13; - - // HTTP 1.1 Entity tag for the object. See - // [https://tools.ietf.org/html/rfc7232#section-2.3][RFC 7232 §2.3]. - // Attempting to set this field will result in an error. - string etag = 14; - - // The modification time of the object metadata. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp updated = 15; - - // Storage class of the object. - string storage_class = 16; - - // Cloud KMS Key used to encrypt this object, if the object is encrypted by - // such a key. - string kms_key_name = 17; - - // The time at which the object's storage class was last changed. When the - // object is initially created, it will be set to time_created. - // Attempting to set this field will result in an error. - google.protobuf.Timestamp time_storage_class_updated = 18; - - // Whether an object is under temporary hold. While this flag is set to true, - // the object is protected against deletion and overwrites. A common use case - // of this flag is regulatory investigations where objects need to be retained - // while the investigation is ongoing. Note that unlike event-based hold, - // temporary hold does not impact retention expiration time of an object. - bool temporary_hold = 19; - - // A server-determined value that specifies the earliest time that the - // object's retention period expires. This value is in - // [https://tools.ietf.org/html/rfc3339][RFC 3339] format. - // Note 1: This field is not provided for objects with an active event-based - // hold, since retention expiration is unknown until the hold is removed. - // Note 2: This value can be provided even when temporary hold is set (so that - // the user can reason about policy without having to first unset the - // temporary hold). - google.protobuf.Timestamp retention_expiration_time = 20; - - // User-provided metadata, in key/value pairs. - map metadata = 21; - - // Whether an object is under event-based hold. Event-based hold is a way to - // retain objects until an event occurs, which is signified by the - // hold's release (i.e. this value is set to false). After being released (set - // to false), such objects will be subject to bucket-level retention (if any). - // One sample use case of this flag is for banks to hold loan documents for at - // least 3 years after loan is paid in full. Here, bucket-level retention is 3 - // years and the event is the loan being paid in full. In this example, these - // objects will be held intact for any number of years until the event has - // occurred (event-based hold on the object is released) and then 3 more years - // after that. That means retention duration of the objects begins from the - // moment event-based hold transitioned from true to false. - google.protobuf.BoolValue event_based_hold = 29; - - // The name of the object. Required if not specified by URL parameter. - // Attempting to update this field after the object is created will result in - // an error. - string name = 23; - - // The ID of the object, including the bucket name, object name, and - // generation number. - // Attempting to update this field after the object is created will result in - // an error. - string id = 24; - - // The name of the bucket containing this object. - // Attempting to update this field after the object is created will result in - // an error. - string bucket = 25; - - // The content generation of this object. Used for object versioning. - // Attempting to set this field will result in an error. - int64 generation = 26; - - // The owner of the object. This will always be the uploader of the object. - // Attempting to set this field will result in an error. - Owner owner = 27; - - // Metadata of customer-supplied encryption key, if the object is encrypted by - // such a key. - CustomerEncryption customer_encryption = 28; -} - -// An access-control entry. -message ObjectAccessControl { - // The access permission for the entity. - string role = 1; - - // HTTP 1.1 Entity tag for the access-control entry. - // See [https://tools.ietf.org/html/rfc7232#section-2.3][RFC 7232 §2.3]. - string etag = 2; - - // The ID of the access-control entry. - string id = 3; - - // The name of the bucket. - string bucket = 4; - - // The name of the object, if applied to an object. - string object = 5; - - // The content generation of the object, if applied to an object. - int64 generation = 6; - - // The entity holding the permission, in one of the following forms: - // * `user-{userid}` - // * `user-{email}` - // * `group-{groupid}` - // * `group-{email}` - // * `domain-{domain}` - // * `project-{team-projectid}` - // * `allUsers` - // * `allAuthenticatedUsers` - // Examples: - // * The user `liz@example.com` would be `user-liz@example.com`. - // * The group `example@googlegroups.com` would be - // `group-example@googlegroups.com`. - // * All members of the Google Apps for Business domain `example.com` would be - // `domain-example.com`. - string entity = 7; - - // The ID for the entity, if any. - string entity_id = 8; - - // The email address associated with the entity, if any. - string email = 9; - - // The domain associated with the entity, if any. - string domain = 10; - - // The project team associated with the entity, if any. - ProjectTeam project_team = 11; -} - -// The result of a call to ObjectAccessControls.ListObjectAccessControls. -message ListObjectAccessControlsResponse { - // The list of items. - repeated ObjectAccessControl items = 1; -} - -// The result of a call to Objects.ListObjects -message ListObjectsResponse { - // The list of prefixes of objects matching-but-not-listed up to and including - // the requested delimiter. - repeated string prefixes = 1; - - // The list of items. - repeated Object items = 2; - - // The continuation token, used to page through large result sets. Provide - // this value in a subsequent request to return the next page of results. - string next_page_token = 3; -} - -// Represents the Viewers, Editors, or Owners of a given project. -message ProjectTeam { - // The project number. - string project_number = 1; - - // The team. - string team = 2; -} - -// A subscription to receive Google PubSub notifications. -message ServiceAccount { - // The ID of the notification. - string email_address = 1; -} - -// The owner of a specific resource. -message Owner { - // The entity, in the form `user-`*userId*. - string entity = 1; - - // The ID for the entity. - string entity_id = 2; -} From 55fa9376407239dfccb8b6214e04ac47cc7d5870 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 13:14:16 -0700 Subject: [PATCH 07/87] reformat gcs --- .../main/java/io/grpc/gcs/GcsioClient.java | 82 +++++---- .../src/main/java/io/grpc/gcs/GrpcClient.java | 168 ++++++++++-------- .../src/main/java/io/grpc/gcs/HttpClient.java | 22 +-- .../main/java/io/grpc/gcs/ResultTable.java | 53 +++--- .../src/main/java/io/grpc/gcs/TestMain.java | 3 +- 5 files changed, 168 insertions(+), 160 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 69cbb486..4230e36b 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -17,12 +17,11 @@ import java.nio.channels.SeekableByteChannel; import java.nio.channels.WritableByteChannel; import java.util.Arrays; +import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.List; import java.util.logging.Logger; -import java.util.Random; public class GcsioClient { private static final Logger logger = Logger.getLogger(GcsioClient.class.getName()); @@ -42,14 +41,19 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { logger.warning("Please provide valid --access_token"); } - this.gcsOpts = GoogleCloudStorageOptions.builder() + this.gcsOpts = + GoogleCloudStorageOptions.builder() .setAppName("weiranf-app") .setGrpcEnabled(grpcEnabled) .setStorageRootUrl("https://" + args.host) .setStorageServicePath(args.service_path) .setDirectPathPreffered(args.dp) - .setReadChannelOptions(GoogleCloudStorageReadOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) - .setWriteChannelOptions(AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) + .setReadChannelOptions( + GoogleCloudStorageReadOptions.builder() + .setGrpcChecksumsEnabled(args.checksum) + .build()) + .setWriteChannelOptions( + AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) .build(); } @@ -75,29 +79,31 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep switch (args.method) { case METHOD_READ: for (int i = 0; i < args.threads; i++) { - Runnable task = () -> { - try { - makeMediaRequest(results); - } catch (IOException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeMediaRequest(results); + } catch (IOException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; - case METHOD_WRITE: + case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeWriteRequest(results, finalI); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeWriteRequest(results, finalI); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } - break; + break; default: logger.warning("Please provide valid methods with --method"); } @@ -111,11 +117,10 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep } private void makeMediaRequest(ResultTable results) throws IOException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder() - .setCloudStorageOptions(gcsOpts) - .build() - ); + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); int size = args.size * 1024; @@ -132,19 +137,19 @@ private void makeMediaRequest(ResultTable results) throws IOException { } buff.clear(); readChannel.close(); - //logger.info("time cost for reading bytes: " + dur + "ms"); + // logger.info("time cost for reading bytes: " + dur + "ms"); results.reportResult(dur); } gcsfs.close(); } - private void makeWriteRequest(ResultTable results, int idx) throws IOException, InterruptedException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder() - .setCloudStorageOptions(gcsOpts) - .build() - ); + private void makeWriteRequest(ResultTable results, int idx) + throws IOException, InterruptedException { + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); int size = args.size * 1024; Random rd = new Random(); @@ -170,16 +175,15 @@ private void makeWriteRequest(ResultTable results, int idx) throws IOException, } private void makeRandomMediaRequest(ResultTable results) throws IOException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder() - .setCloudStorageOptions(gcsOpts) - .build() - ); + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); Random r = new Random(); URI uri = URI.create("gs://" + args.bkt + "/" + args.obj); - + GoogleCloudStorageReadOptions readOpts = gcsOpts.getReadChannelOptions(); SeekableByteChannel reader = gcsfs.open(uri, readOpts); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 3c5b757b..9aefa9bb 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -3,53 +3,51 @@ import static io.grpc.gcs.Args.METHOD_RANDOM; import static io.grpc.gcs.Args.METHOD_READ; import static io.grpc.gcs.Args.METHOD_WRITE; -import static io.grpc.gcs.Args.DEFAULT_HOST; import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.ByteString; import com.google.storage.v2.ChecksummedData; +import com.google.storage.v2.Object; import com.google.storage.v2.ReadObjectRequest; import com.google.storage.v2.ReadObjectResponse; -import com.google.storage.v2.WriteObjectRequest; -import com.google.storage.v2.WriteObjectSpec; -import com.google.storage.v2.WriteObjectResponse; -import com.google.storage.v2.Object; import com.google.storage.v2.ServiceConstants.Values; import com.google.storage.v2.StorageGrpc; import com.google.storage.v2.StorageGrpc.StorageBlockingStub; -import com.google.protobuf.ByteString; +import com.google.storage.v2.WriteObjectRequest; +import com.google.storage.v2.WriteObjectResponse; +import com.google.storage.v2.WriteObjectSpec; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.MethodDescriptor; import io.grpc.Status; import io.grpc.alts.ComputeEngineChannelBuilder; import io.grpc.auth.MoreCallCredentials; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.StreamObserver; -import io.grpc.MethodDescriptor; import java.io.IOException; -import java.lang.reflect.Field; import java.io.InputStream; +import java.lang.reflect.Field; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.Iterator; -import java.util.List; import java.util.logging.Logger; -import java.util.NoSuchElementException; -import java.util.Random; public class GrpcClient { private static final Logger logger = Logger.getLogger(GrpcClient.class.getName()); // ZeroCopy version of GetObjectMedia Method private static final ZeroCopyMessageMarshaller ReadObjectResponseMarshaller = - new ZeroCopyMessageMarshaller(ReadObjectResponse.getDefaultInstance()); + new ZeroCopyMessageMarshaller(ReadObjectResponse.getDefaultInstance()); private static final MethodDescriptor readObjectMethod = - StorageGrpc.getReadObjectMethod() - .toBuilder().setResponseMarshaller(ReadObjectResponseMarshaller) - .build(); + StorageGrpc.getReadObjectMethod().toBuilder() + .setResponseMarshaller(ReadObjectResponseMarshaller) + .build(); private final boolean useZeroCopy; private ManagedChannel[] channels; @@ -75,16 +73,19 @@ public GrpcClient(Args args) throws IOException { ManagedChannelBuilder channelBuilder; if (args.dp) { - ComputeEngineChannelBuilder gceChannelBuilder = ComputeEngineChannelBuilder.forAddress(args.host, args.port); + ComputeEngineChannelBuilder gceChannelBuilder = + ComputeEngineChannelBuilder.forAddress(args.host, args.port); ImmutableMap pickFirstStrategy = ImmutableMap.of("pick_first", ImmutableMap.of()); ImmutableMap childPolicy = - ImmutableMap.of("childPolicy", ImmutableList.of(pickFirstStrategy)); + ImmutableMap.of( + "childPolicy", ImmutableList.of(pickFirstStrategy)); ImmutableMap grpcLbPolicy = ImmutableMap.of("grpclb", childPolicy); ImmutableMap loadBalancingConfig = - ImmutableMap.of("loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); @@ -94,7 +95,8 @@ public GrpcClient(Args args) throws IOException { delegateField = ComputeEngineChannelBuilder.class.getDeclaredField("delegate"); delegateField.setAccessible(true); - NettyChannelBuilder delegateBuilder = (NettyChannelBuilder) delegateField.get(gceChannelBuilder); + NettyChannelBuilder delegateBuilder = + (NettyChannelBuilder) delegateField.get(gceChannelBuilder); delegateBuilder.flowControlWindow(args.flowControlWindow); } catch (NoSuchFieldException | IllegalAccessException e) { e.printStackTrace(); @@ -103,7 +105,8 @@ public GrpcClient(Args args) throws IOException { } channelBuilder = gceChannelBuilder; } else { - NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(args.host, args.port); + NettyChannelBuilder nettyChannelBuilder = + NettyChannelBuilder.forAddress(args.host, args.port); if (args.flowControlWindow > 0) { nettyChannelBuilder.flowControlWindow(args.flowControlWindow); } @@ -166,13 +169,14 @@ public void startCalls(ResultTable results) throws InterruptedException { case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeInsertRequest(this.channels[finalI], results, finalI); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeInsertRequest(this.channels[finalI], results, finalI); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -189,22 +193,27 @@ public void startCalls(ResultTable results) throws InterruptedException { } private void makeReadObjectRequest(ManagedChannel channel, ResultTable results) { - StorageGrpc.StorageBlockingStub blockingStub = - StorageGrpc.newBlockingStub(channel); + StorageGrpc.StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { - blockingStub = blockingStub.withCallCredentials( - MoreCallCredentials.from(creds)); + blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } ReadObjectRequest readRequest = - ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(args.obj).build(); - byte[] scratch = new byte[4*1024*1024]; + ReadObjectRequest.newBuilder() + .setBucket(toV2BucketName(args.bkt)) + .setObject(args.obj) + .build(); + byte[] scratch = new byte[4 * 1024 * 1024]; for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); Iterator resIterator; if (useZeroCopy) { - resIterator = io.grpc.stub.ClientCalls.blockingServerStreamingCall( - blockingStub.getChannel(), readObjectMethod, blockingStub.getCallOptions(), readRequest); + resIterator = + io.grpc.stub.ClientCalls.blockingServerStreamingCall( + blockingStub.getChannel(), + readObjectMethod, + blockingStub.getCallOptions(), + readRequest); } else { resIterator = blockingStub.readObject(readRequest); } @@ -237,11 +246,9 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results) } private void makeRandomReadRequest(ManagedChannel channel, ResultTable results) { - StorageBlockingStub blockingStub = - StorageGrpc.newBlockingStub(channel); + StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { - blockingStub = blockingStub.withCallCredentials( - MoreCallCredentials.from(creds)); + blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } ReadObjectRequest.Builder reqBuilder = @@ -264,21 +271,27 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results) itr++; ReadObjectResponse res = resIterator.next(); bytesRead += res.getChecksummedData().getSerializedSize(); - //logger.info("result: " + res.getChecksummedData()); + // logger.info("result: " + res.getChecksummedData()); } long dur = System.currentTimeMillis() - start; logger.info("time cost for getObjectMedia: " + dur + "ms"); logger.info("total iterations: " + itr); - logger.info("start pos: " + offset + ", read lenth: " + buffSize + ", total KB read: " + bytesRead / 1024); + logger.info( + "start pos: " + + offset + + ", read lenth: " + + buffSize + + ", total KB read: " + + bytesRead / 1024); results.reportResult(dur); } } - private void makeInsertRequest(ManagedChannel channel, ResultTable results, int idx) throws InterruptedException { + private void makeInsertRequest(ManagedChannel channel, ResultTable results, int idx) + throws InterruptedException { StorageGrpc.StorageStub asyncStub = StorageGrpc.newStub(channel); if (creds != null) { - asyncStub = asyncStub.withCallCredentials( - MoreCallCredentials.from(creds)); + asyncStub = asyncStub.withCallCredentials(MoreCallCredentials.from(creds)); } int totalBytes = args.size * 1024; @@ -289,41 +302,41 @@ private void makeInsertRequest(ManagedChannel channel, ResultTable results, int boolean isLast = false; final CountDownLatch finishLatch = new CountDownLatch(1); - StreamObserver responseObserver = new StreamObserver() { - long start = System.currentTimeMillis(); + StreamObserver responseObserver = + new StreamObserver() { + long start = System.currentTimeMillis(); - @Override - public void onNext(WriteObjectResponse value) { - } + @Override + public void onNext(WriteObjectResponse value) {} - @Override - public void onError(Throwable t) { - logger.warning("InsertObject failed with: " + Status.fromThrowable(t)); - finishLatch.countDown(); - } + @Override + public void onError(Throwable t) { + logger.warning("InsertObject failed with: " + Status.fromThrowable(t)); + finishLatch.countDown(); + } - @Override - public void onCompleted() { - long dur = System.currentTimeMillis() - start; - results.reportResult(dur); - if (dur < 1000) { - try { - Thread.sleep(1000 - dur); // Avoid limit of 1 qps for updating the same object - } catch (InterruptedException e) { - e.printStackTrace(); + @Override + public void onCompleted() { + long dur = System.currentTimeMillis() - start; + results.reportResult(dur); + if (dur < 1000) { + try { + Thread.sleep(1000 - dur); // Avoid limit of 1 qps for updating the same object + } catch (InterruptedException e) { + e.printStackTrace(); + finishLatch.countDown(); + } + } finishLatch.countDown(); } - } - finishLatch.countDown(); - } - }; + }; StreamObserver requestObserver = asyncStub.writeObject(responseObserver); while (offset < totalBytes) { int add; if (offset + Values.MAX_WRITE_CHUNK_BYTES_VALUE <= totalBytes) { - add = Values.MAX_WRITE_CHUNK_BYTES_VALUE; + add = Values.MAX_WRITE_CHUNK_BYTES_VALUE; } else { add = totalBytes - offset; } @@ -331,7 +344,8 @@ public void onCompleted() { isLast = true; } - WriteObjectRequest req = getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), idx); + WriteObjectRequest req = + getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), idx); requestObserver.onNext(req); if (finishLatch.getCount() == 0) { logger.warning("Stream completed before finishing sending requests"); @@ -346,17 +360,19 @@ public void onCompleted() { logger.warning("insertObject cannot finish within 20 minutes"); } } - } - private WriteObjectRequest getWriteRequest(boolean first, boolean last, int offset, ByteString bytes, int idx) { + private WriteObjectRequest getWriteRequest( + boolean first, boolean last, int offset, ByteString bytes, int idx) { WriteObjectRequest.Builder builder = WriteObjectRequest.newBuilder(); if (first) { builder.setWriteObjectSpec( - WriteObjectSpec.newBuilder().setResource( - Object.newBuilder().setBucket(toV2BucketName(args.bkt)).setName(args.obj + "_" + idx) - ).build() - ); + WriteObjectSpec.newBuilder() + .setResource( + Object.newBuilder() + .setBucket(toV2BucketName(args.bkt)) + .setName(args.obj + "_" + idx)) + .build()); } builder.setChecksummedData(ChecksummedData.newBuilder().setContent(bytes).build()); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java index f3334373..c407668d 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java @@ -3,25 +3,20 @@ import static io.grpc.gcs.Args.METHOD_RANDOM; import static io.grpc.gcs.Args.METHOD_READ; import static io.grpc.gcs.Args.METHOD_WRITE; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.cloud.ReadChannel; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Bucket; -import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; - public class HttpClient { private static final Logger logger = Logger.getLogger(HttpClient.class.getName()); @@ -73,11 +68,11 @@ public void makeMediaRequest(ResultTable results) { for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); byte[] content = client.readAllBytes(blobId); - //String contentString = new String(content, UTF_8); - //logger.info("contentString: " + contentString); + // String contentString = new String(content, UTF_8); + // logger.info("contentString: " + contentString); long dur = System.currentTimeMillis() - start; - //logger.info("time cost for readAllBytes: " + dur + "ms"); - //logger.info("total KB received: " + content.length/1024); + // logger.info("time cost for readAllBytes: " + dur + "ms"); + // logger.info("total KB received: " + content.length/1024); results.reportResult(dur); } } @@ -98,7 +93,7 @@ public void makeRandomMediaRequest(ResultTable results) throws IOException { if (buff.remaining() > 0) { logger.warning("Got remaining bytes: " + buff.remaining()); } - logger.info("total KB received: " + buff.position()/1024); + logger.info("total KB received: " + buff.position() / 1024); logger.info("time cost for random reading: " + dur + "ms"); buff.clear(); results.reportResult(dur); @@ -112,10 +107,7 @@ public void makeInsertRequest(ResultTable results) { BlobId blobId = BlobId.of(args.bkt, args.obj); for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); - client.create( - BlobInfo.newBuilder(blobId).build(), - data - ); + client.create(BlobInfo.newBuilder(blobId).build(), data); long dur = System.currentTimeMillis() - start; logger.info("time cost for creating blob: " + dur + "ms"); results.reportResult(dur); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java index e85c11b8..ec7a1928 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java @@ -1,17 +1,11 @@ package io.grpc.gcs; import com.google.gson.Gson; -import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; -import java.security.Security; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.logging.LogManager; -import java.util.logging.Logger; -import org.conscrypt.Conscrypt; - public class ResultTable { private Args args; @@ -32,7 +26,7 @@ public void start() { public void stop() { synchronized (this) { - endTime = System.currentTimeMillis(); + endTime = System.currentTimeMillis(); } } @@ -77,27 +71,28 @@ public void printResult() throws IOException { gson.toJson(benchmarkResult, writer); writer.close(); } - System.out.println(String.format( - "****** Test Results [client: %s, method: %s, size: %d, threads: %d, dp: %s, calls: %d]: \n" - + "\t\tMin\tp5\tp10\tp25\tp50\tp75\tp90\tp99\tMax\tTotal\n" - + " Time(ms)\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", - args.client, - args.method, - args.size, - args.threads, - args.dp, - n, - results.get(0), - results.get((int) (n * 0.05)), - results.get((int) (n * 0.1)), - results.get((int) (n * 0.25)), - results.get((int) (n * 0.50)), - results.get((int) (n * 0.75)), - results.get((int) (n * 0.90)), - results.get((int) (n * 0.99)), - results.get(n - 1), - totalDur - )); - } + System.out.println( + String.format( + "****** Test Results [client: %s, method: %s, size: %d, threads: %d, dp: %s, calls:" + + " %d]: \n" + + "\t\tMin\tp5\tp10\tp25\tp50\tp75\tp90\tp99\tMax\tTotal\n" + + " Time(ms)\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", + args.client, + args.method, + args.size, + args.threads, + args.dp, + n, + results.get(0), + results.get((int) (n * 0.05)), + results.get((int) (n * 0.1)), + results.get((int) (n * 0.25)), + results.get((int) (n * 0.50)), + results.get((int) (n * 0.75)), + results.get((int) (n * 0.90)), + results.get((int) (n * 0.99)), + results.get(n - 1), + totalDur)); + } } } diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/TestMain.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/TestMain.java index 73e9df1e..b5122087 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/TestMain.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/TestMain.java @@ -22,7 +22,8 @@ public static void main(String[] args) throws Exception { if (a.conscrypt) { Security.insertProviderAt(Conscrypt.newProvider(), 1); } else if (a.conscrypt_notm) { - Security.insertProviderAt(Conscrypt.newProviderBuilder().provideTrustManager(false).build(), 1); + Security.insertProviderAt( + Conscrypt.newProviderBuilder().provideTrustManager(false).build(), 1); } ResultTable results = new ResultTable(a); long start = 0; From 97f039c9003931dc07d75131f728c31b4d8f8f37 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 15:21:53 -0700 Subject: [PATCH 08/87] Added td option --- .../gcs/src/main/java/io/grpc/gcs/Args.java | 3 ++ .../src/main/java/io/grpc/gcs/GrpcClient.java | 37 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java index 9faee081..529909ed 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java @@ -26,6 +26,7 @@ public class Args { final String access_token; final String bkt, obj; final boolean dp; + final boolean td; final int size; final int buffSize; final String method; @@ -56,6 +57,7 @@ public class Args { parser.addArgument("--bkt").type(String.class).setDefault("gcs-grpc-team-weiranf"); parser.addArgument("--obj").type(String.class).setDefault("a"); parser.addArgument("--dp").type(Boolean.class).setDefault(false); + parser.addArgument("--td").type(Boolean.class).setDefault(false); parser.addArgument("--size").type(Integer.class).setDefault(0); parser.addArgument("--buffSize").type(Integer.class).setDefault(0); parser.addArgument("--method").type(String.class).setDefault(METHOD_READ); @@ -82,6 +84,7 @@ public class Args { bkt = ns.getString("bkt"); obj = ns.getString("obj"); dp = ns.getBoolean("dp"); + td = ns.getBoolean("td"); size = ns.getInt("size"); buffSize = ns.getInt("buffSize"); method = ns.getString("method"); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 9aefa9bb..b4c24948 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -71,23 +71,29 @@ public GrpcClient(Args args) throws IOException { logger.warning("Please provide valid --access_token"); } + String target = args.host; + if (args.td) { + target = "google-c2p:///" + target; + } + ManagedChannelBuilder channelBuilder; if (args.dp) { ComputeEngineChannelBuilder gceChannelBuilder = - ComputeEngineChannelBuilder.forAddress(args.host, args.port); - - ImmutableMap pickFirstStrategy = - ImmutableMap.of("pick_first", ImmutableMap.of()); - ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(pickFirstStrategy)); - ImmutableMap grpcLbPolicy = - ImmutableMap.of("grpclb", childPolicy); - ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); - - gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); + ComputeEngineChannelBuilder.forAddress(target, args.port); + + if (!args.td) { + ImmutableMap pickFirstStrategy = + ImmutableMap.of("pick_first", ImmutableMap.of()); + ImmutableMap childPolicy = + ImmutableMap.of( + "childPolicy", ImmutableList.of(pickFirstStrategy)); + ImmutableMap grpcLbPolicy = + ImmutableMap.of("grpclb", childPolicy); + ImmutableMap loadBalancingConfig = + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); + } if (args.flowControlWindow > 0) { Field delegateField = null; @@ -105,8 +111,7 @@ public GrpcClient(Args args) throws IOException { } channelBuilder = gceChannelBuilder; } else { - NettyChannelBuilder nettyChannelBuilder = - NettyChannelBuilder.forAddress(args.host, args.port); + NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(target, args.port); if (args.flowControlWindow > 0) { nettyChannelBuilder.flowControlWindow(args.flowControlWindow); } From b0f060626e8b445e6db825e68bdec23974b35996 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 15:30:51 -0700 Subject: [PATCH 09/87] target --- .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index b4c24948..6cc67959 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -79,7 +79,9 @@ public GrpcClient(Args args) throws IOException { ManagedChannelBuilder channelBuilder; if (args.dp) { ComputeEngineChannelBuilder gceChannelBuilder = - ComputeEngineChannelBuilder.forAddress(target, args.port); + args.td + ? ComputeEngineChannelBuilder.forTarget(target) + : ComputeEngineChannelBuilder.forAddress(target, args.port); if (!args.td) { ImmutableMap pickFirstStrategy = From 22aa69e720cdef997c77e1f64c90cabac64a82c5 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 15:35:47 -0700 Subject: [PATCH 10/87] gRPC 1.40.1 --- end2end-test-examples/gcs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 1d1a40da..a7b99289 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -10,7 +10,7 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 def gcsioVersion = '2.2.3-SNAPSHOT' -def grpcVersion = '1.39.0' +def grpcVersion = '1.40.1' def protobufVersion = '3.17.3' def protocVersion = protobufVersion def conscryptVersion = '2.5.1' From 16659b6e82162ec93c59165e80bd80e8ec8109d2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 15:38:02 -0700 Subject: [PATCH 11/87] added target --- .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 6cc67959..55060606 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -113,7 +113,10 @@ public GrpcClient(Args args) throws IOException { } channelBuilder = gceChannelBuilder; } else { - NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(target, args.port); + NettyChannelBuilder nettyChannelBuilder = + args.td + ? NettyChannelBuilder.forTarget(target) + : NettyChannelBuilder.forAddress(target, args.port); if (args.flowControlWindow > 0) { nettyChannelBuilder.flowControlWindow(args.flowControlWindow); } From d6efd569a204b9b2b89b339d7bcf753e7f92bd73 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 26 Aug 2021 15:43:11 -0700 Subject: [PATCH 12/87] add xds --- end2end-test-examples/gcs/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index a7b99289..a6485ce6 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -27,6 +27,7 @@ dependencies { compile "io.grpc:grpc-netty-shaded:${grpcVersion}" compile "io.grpc:grpc-auth:${grpcVersion}" compile "io.grpc:grpc-alts:${grpcVersion}" + compile "io.grpc:grpc-xds:${grpcVersion}" compile "org.conscrypt:conscrypt-openjdk-uber:${conscryptVersion}" compile "com.google.protobuf:protobuf-java-util:${protobufVersion}" compile "com.google.api.grpc:grpc-google-cloud-storage-v2:latest.release" From 35ed43f86f4ac3ee49fe9637fe6b98351f4c28f4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 7 Oct 2021 13:59:37 -0700 Subject: [PATCH 13/87] Update gcsio to 2.2.3 --- end2end-test-examples/gcs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index a6485ce6..19add907 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -9,7 +9,7 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 -def gcsioVersion = '2.2.3-SNAPSHOT' +def gcsioVersion = '2.2.3' def grpcVersion = '1.40.1' def protobufVersion = '3.17.3' def protocVersion = protobufVersion From 41d3c15e04ba4a71f96829cb70c5160493533d58 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 13 Oct 2021 15:15:33 -0700 Subject: [PATCH 14/87] Upgrade guava to 31.0.1-jre Without this, it will have 31.0.1-android which causes the following exception. ``` Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.cache.CacheBuilder.expireAfterWrite(Ljava/time/Duration;)Lcom/google/common/cache/CacheBuilder; at com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.(GoogleCloudStorageImpl.java:202) at com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.(GoogleCloudStorageImpl.java:317) at com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.(GoogleCloudStorageImpl.java:278) at com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem.(GoogleCloudStorageFileSystem.java:150) at io.grpc.gcs.GcsioClient.makeWriteRequest(GcsioClient.java:152) at io.grpc.gcs.GcsioClient.startCalls(GcsioClient.java:67) at io.grpc.gcs.TestMain.main(TestMain.java:47) ``` --- end2end-test-examples/gcs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 19add907..86903cb8 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -38,7 +38,7 @@ dependencies { compile "com.google.cloud:google-cloud-storage:1.113.9" compile "com.google.cloud.bigdataoss:gcsio:${gcsioVersion}" compile "com.google.cloud.bigdataoss:bigdataoss-parent:${gcsioVersion}" - compile "com.google.guava:guava:30.1.1-jre" + compile "com.google.guava:guava:31.0.1-jre" testCompile group: 'junit', name: 'junit', version: '4.12' } From c759319c706eef38b524ea3bb0c7c2d8ddff052c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 26 Oct 2021 16:03:34 -0700 Subject: [PATCH 15/87] Added round-robin option --- .../gcs/src/main/java/io/grpc/gcs/Args.java | 3 +++ .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java index 529909ed..065c3368 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java @@ -26,6 +26,7 @@ public class Args { final String access_token; final String bkt, obj; final boolean dp; + final boolean rr; final boolean td; final int size; final int buffSize; @@ -57,6 +58,7 @@ public class Args { parser.addArgument("--bkt").type(String.class).setDefault("gcs-grpc-team-weiranf"); parser.addArgument("--obj").type(String.class).setDefault("a"); parser.addArgument("--dp").type(Boolean.class).setDefault(false); + parser.addArgument("--rr").type(Boolean.class).setDefault(false); parser.addArgument("--td").type(Boolean.class).setDefault(false); parser.addArgument("--size").type(Integer.class).setDefault(0); parser.addArgument("--buffSize").type(Integer.class).setDefault(0); @@ -84,6 +86,7 @@ public class Args { bkt = ns.getString("bkt"); obj = ns.getString("obj"); dp = ns.getBoolean("dp"); + rr = ns.getBoolean("rr"); td = ns.getBoolean("td"); size = ns.getInt("size"); buffSize = ns.getInt("buffSize"); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 55060606..c5af2287 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -83,7 +83,7 @@ public GrpcClient(Args args) throws IOException { ? ComputeEngineChannelBuilder.forTarget(target) : ComputeEngineChannelBuilder.forAddress(target, args.port); - if (!args.td) { + if (!args.td && !args.rr) { ImmutableMap pickFirstStrategy = ImmutableMap.of("pick_first", ImmutableMap.of()); ImmutableMap childPolicy = @@ -125,8 +125,17 @@ public GrpcClient(Args args) throws IOException { // Create the same number of channels as the number of threads. this.channels = new ManagedChannel[args.threads]; - for (int i = 0; i < args.threads; i++) { - channels[i] = channelBuilder.build(); + if (args.rr) { + // For round-robin, all threads share the same channel. + ManagedChannel singleChannel = channelBuilder.build(); + for (int i = 0; i < args.threads; i++) { + channels[i] = singleChannel; + } + } else { + // For pick-first, each thread has its own unique channel. + for (int i = 0; i < args.threads; i++) { + channels[i] = channelBuilder.build(); + } } if (args.zeroCopy == 0) { From 37941c1451c33405851842f54710ed2cd815c11f Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 26 Oct 2021 16:20:42 -0700 Subject: [PATCH 16/87] explicit RR --- .../src/main/java/io/grpc/gcs/GrpcClient.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index c5af2287..7fb3de5c 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -83,18 +83,32 @@ public GrpcClient(Args args) throws IOException { ? ComputeEngineChannelBuilder.forTarget(target) : ComputeEngineChannelBuilder.forAddress(target, args.port); - if (!args.td && !args.rr) { - ImmutableMap pickFirstStrategy = - ImmutableMap.of("pick_first", ImmutableMap.of()); - ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(pickFirstStrategy)); - ImmutableMap grpcLbPolicy = - ImmutableMap.of("grpclb", childPolicy); - ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); - gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); + if (!args.td) { + if (args.rr) { + ImmutableMap pickFirstStrategy = + ImmutableMap.of("round_robin", ImmutableMap.of()); + ImmutableMap childPolicy = + ImmutableMap.of( + "childPolicy", ImmutableList.of(pickFirstStrategy)); + ImmutableMap grpcLbPolicy = + ImmutableMap.of("grpclb", childPolicy); + ImmutableMap loadBalancingConfig = + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); + } else { + ImmutableMap pickFirstStrategy = + ImmutableMap.of("pick_first", ImmutableMap.of()); + ImmutableMap childPolicy = + ImmutableMap.of( + "childPolicy", ImmutableList.of(pickFirstStrategy)); + ImmutableMap grpcLbPolicy = + ImmutableMap.of("grpclb", childPolicy); + ImmutableMap loadBalancingConfig = + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); + } } if (args.flowControlWindow > 0) { From 3a09f2877960fa3f20ce65588ce158570f8f62c0 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 26 Oct 2021 16:36:23 -0700 Subject: [PATCH 17/87] test --- .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 7fb3de5c..709519d0 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -139,18 +139,19 @@ public GrpcClient(Args args) throws IOException { // Create the same number of channels as the number of threads. this.channels = new ManagedChannel[args.threads]; + /* if (args.rr) { // For round-robin, all threads share the same channel. ManagedChannel singleChannel = channelBuilder.build(); for (int i = 0; i < args.threads; i++) { channels[i] = singleChannel; } - } else { + } else {*/ // For pick-first, each thread has its own unique channel. for (int i = 0; i < args.threads; i++) { channels[i] = channelBuilder.build(); } - } + //} if (args.zeroCopy == 0) { useZeroCopy = ZeroCopyReadinessChecker.isReady(); From 287e7ff0a695cb25d3b9c2a6e863bc92f07c3409 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 27 Oct 2021 15:56:02 -0700 Subject: [PATCH 18/87] clean up --- .../src/main/java/io/grpc/gcs/GrpcClient.java | 42 +++++++------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 709519d0..f0d2e0a3 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -84,31 +84,18 @@ public GrpcClient(Args args) throws IOException { : ComputeEngineChannelBuilder.forAddress(target, args.port); if (!args.td) { - if (args.rr) { - ImmutableMap pickFirstStrategy = - ImmutableMap.of("round_robin", ImmutableMap.of()); - ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(pickFirstStrategy)); - ImmutableMap grpcLbPolicy = - ImmutableMap.of("grpclb", childPolicy); - ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); - gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); - } else { - ImmutableMap pickFirstStrategy = - ImmutableMap.of("pick_first", ImmutableMap.of()); - ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(pickFirstStrategy)); - ImmutableMap grpcLbPolicy = - ImmutableMap.of("grpclb", childPolicy); - ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); - gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); - } + String policy = args.rr ? "round_robin" : "pick_first"; + ImmutableMap policyStrategy = + ImmutableMap.of(policy, ImmutableMap.of()); + ImmutableMap childPolicy = + ImmutableMap.of( + "childPolicy", ImmutableList.of(policyStrategy)); + ImmutableMap grpcLbPolicy = + ImmutableMap.of("grpclb", childPolicy); + ImmutableMap loadBalancingConfig = + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); } if (args.flowControlWindow > 0) { @@ -139,19 +126,18 @@ public GrpcClient(Args args) throws IOException { // Create the same number of channels as the number of threads. this.channels = new ManagedChannel[args.threads]; - /* if (args.rr) { // For round-robin, all threads share the same channel. ManagedChannel singleChannel = channelBuilder.build(); for (int i = 0; i < args.threads; i++) { channels[i] = singleChannel; } - } else {*/ + } else { // For pick-first, each thread has its own unique channel. for (int i = 0; i < args.threads; i++) { channels[i] = channelBuilder.build(); } - //} + } if (args.zeroCopy == 0) { useZeroCopy = ZeroCopyReadinessChecker.isReady(); From 65801a54baea01495b53c8e953c91c1ade2f7aa5 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 8 Nov 2021 12:52:07 -0800 Subject: [PATCH 19/87] Added host2 for gRPC --- end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java | 3 +++ .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java index 065c3368..7d9099d6 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java @@ -21,6 +21,7 @@ public class Args { final int calls; final String cookie; final String host; + final String host2; final int port; final String service_path; final String access_token; @@ -52,6 +53,7 @@ public class Args { parser.addArgument("--calls").type(Integer.class).setDefault(1); parser.addArgument("--cookie").type(String.class).setDefault(""); parser.addArgument("--host").type(String.class).setDefault(DEFAULT_HOST); + parser.addArgument("--host2").type(String.class).setDefault(""); parser.addArgument("--port").type(Integer.class).setDefault(PORT); parser.addArgument("--service_path").type(String.class).setDefault("storage/v1/"); parser.addArgument("--access_token").type(String.class).setDefault(""); @@ -80,6 +82,7 @@ public class Args { calls = ns.getInt("calls"); cookie = ns.getString("cookie"); host = ns.getString("host"); + host2 = ns.getString("host2"); port = ns.getInt("port"); service_path = ns.getString("service_path"); access_token = ns.getString("access_token"); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 4230e36b..a483e6e7 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -10,6 +10,7 @@ import com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions; import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions; import com.google.cloud.hadoop.util.AsyncWriteChannelOptions; +import com.google.common.base.Strings; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; @@ -50,6 +51,7 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { .setDirectPathPreffered(args.dp) .setReadChannelOptions( GoogleCloudStorageReadOptions.builder() + .setGrpcServerAddress(Strings.isNullOrEmpty(args.host2) ? null : args.host2) .setGrpcChecksumsEnabled(args.checksum) .build()) .setWriteChannelOptions( From f1e00d62644673e426acf0d08757e26bc1d754e3 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 24 Nov 2021 12:26:23 -0800 Subject: [PATCH 20/87] Deps upgrade --- end2end-test-examples/gcs/build.gradle | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 86903cb8..b8772d02 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -9,8 +9,8 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 -def gcsioVersion = '2.2.3' -def grpcVersion = '1.40.1' +def gcsioVersion = '2.2.5-SNAPSHOT' +def grpcVersion = '1.42.1' def protobufVersion = '3.17.3' def protocVersion = protobufVersion def conscryptVersion = '2.5.1' @@ -21,24 +21,27 @@ repositories { } dependencies { + compile "io.grpc:grpc-alts:${grpcVersion}" + compile "io.grpc:grpc-api:${grpcVersion}" + compile "io.grpc:grpc-auth:${grpcVersion}" + compile "io.grpc:grpc-context:${grpcVersion}" + compile "io.grpc:grpc-netty-shaded:${grpcVersion}" compile "io.grpc:grpc-protobuf:${grpcVersion}" + compile "io.grpc:grpc-protobuf-lite:${grpcVersion}" compile "io.grpc:grpc-stub:${grpcVersion}" compile "io.grpc:grpc-testing:${grpcVersion}" - compile "io.grpc:grpc-netty-shaded:${grpcVersion}" - compile "io.grpc:grpc-auth:${grpcVersion}" - compile "io.grpc:grpc-alts:${grpcVersion}" compile "io.grpc:grpc-xds:${grpcVersion}" compile "org.conscrypt:conscrypt-openjdk-uber:${conscryptVersion}" compile "com.google.protobuf:protobuf-java-util:${protobufVersion}" compile "com.google.api.grpc:grpc-google-cloud-storage-v2:latest.release" compile "com.google.api.grpc:proto-google-iam-v1:latest.release" compile "com.google.api.grpc:proto-google-common-protos:latest.release" - compile "net.sourceforge.argparse4j:argparse4j:0.8.1" - compile "com.google.auth:google-auth-library-oauth2-http:0.22.2" - compile "com.google.cloud:google-cloud-storage:1.113.9" + compile "com.google.auth:google-auth-library-oauth2-http:latest.release" + compile "com.google.cloud:google-cloud-storage:latest.release" compile "com.google.cloud.bigdataoss:gcsio:${gcsioVersion}" compile "com.google.cloud.bigdataoss:bigdataoss-parent:${gcsioVersion}" compile "com.google.guava:guava:31.0.1-jre" + compile "net.sourceforge.argparse4j:argparse4j:0.9.0" testCompile group: 'junit', name: 'junit', version: '4.12' } From 08107a606a7f3db3c744553f881912e999bb8e08 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 2 Dec 2021 15:52:23 -0800 Subject: [PATCH 21/87] Fix typo --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index a483e6e7..aff34bff 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -48,7 +48,7 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { .setGrpcEnabled(grpcEnabled) .setStorageRootUrl("https://" + args.host) .setStorageServicePath(args.service_path) - .setDirectPathPreffered(args.dp) + .setDirectPathPreferred(args.dp) .setReadChannelOptions( GoogleCloudStorageReadOptions.builder() .setGrpcServerAddress(Strings.isNullOrEmpty(args.host2) ? null : args.host2) From 5f094e9e37b14355853e826e5584a2f34e7d7a53 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 3 Dec 2021 15:42:19 -0800 Subject: [PATCH 22/87] Support TD for GCSIO --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index aff34bff..dbf66405 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -48,6 +48,7 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { .setGrpcEnabled(grpcEnabled) .setStorageRootUrl("https://" + args.host) .setStorageServicePath(args.service_path) + .setTrafficDirectorEnabled(args.td) .setDirectPathPreferred(args.dp) .setReadChannelOptions( GoogleCloudStorageReadOptions.builder() From 99b9c190d3d5705d3c1e2d4b1ff45640bcee3e76 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 9 Dec 2021 15:07:01 -0800 Subject: [PATCH 23/87] Consolidate channel pool config in the GcpChannelPoolOptions. Start moving channel pool related configuration into GcpChannelPoolOptions inside GcpManagedChannelOptions. Deprecate previous locations of pool config. Currently, we have several places to define max pool size, and we have a new pool related config options to come in the following PRs. We want to have a single place to define pool-related configuration. We will keep ApiConfig as a configuration for API methods only and move everything else into GcpManagedChannelOptions. --- .../google/cloud/grpc/GcpManagedChannel.java | 44 +++++++-- .../cloud/grpc/GcpManagedChannelBuilder.java | 5 +- .../cloud/grpc/GcpManagedChannelOptions.java | 97 +++++++++++++++++++ .../google/grpc/gcp/proto/grpc_gcp.proto | 7 +- .../cloud/grpc/GcpManagedChannelTest.java | 46 +++++++++ 5 files changed, 187 insertions(+), 12 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 44bfd7ff..b3b5095c 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -59,8 +59,8 @@ public class GcpManagedChannel extends ManagedChannel { private static final Logger logger = Logger.getLogger(GcpManagedChannel.class.getName()); static final AtomicInteger channelPoolIndex = new AtomicInteger(); - private static final int DEFAULT_MAX_CHANNEL = 10; - private static final int DEFAULT_MAX_STREAM = 100; + static final int DEFAULT_MAX_CHANNEL = 10; + static final int DEFAULT_MAX_STREAM = 100; private final ManagedChannelBuilder delegateChannelBuilder; private final GcpManagedChannelOptions options; @@ -141,21 +141,17 @@ public class GcpManagedChannel extends ManagedChannel { * @param options the options for GcpManagedChannel. */ public GcpManagedChannel( - ManagedChannelBuilder delegateChannelBuilder, - ApiConfig apiConfig, - int poolSize, - GcpManagedChannelOptions options) { + ManagedChannelBuilder delegateChannelBuilder, + ApiConfig apiConfig, + GcpManagedChannelOptions options) { loadApiConfig(apiConfig); - if (poolSize != 0) { - this.maxSize = poolSize; - } this.delegateChannelBuilder = delegateChannelBuilder; this.options = options; initOptions(); if (options.getResiliencyOptions() != null) { fallbackEnabled = options.getResiliencyOptions().isNotReadyFallbackEnabled(); unresponsiveDetectionEnabled = - options.getResiliencyOptions().isUnresponsiveDetectionEnabled(); + options.getResiliencyOptions().isUnresponsiveDetectionEnabled(); unresponsiveMs = options.getResiliencyOptions().getUnresponsiveDetectionMs(); unresponsiveDropCount = options.getResiliencyOptions().getUnresponsiveDetectionDroppedCount(); } else { @@ -166,7 +162,35 @@ public GcpManagedChannel( } } + /** + * Constructor for GcpManagedChannel. + * Deprecated. Use the one without the poolSize and set the maximum pool size in options. However, note that if + * setting the pool size from options then concurrent streams low watermark (even the default one) will be also taken + * from the options and not apiConfig. + * + * @param delegateChannelBuilder the underlying delegate ManagedChannelBuilder. + * @param apiConfig the ApiConfig object for configuring GcpManagedChannel. + * @param poolSize maximum number of channels the pool can have. + * @param options the options for GcpManagedChannel. + */ + @Deprecated + public GcpManagedChannel( + ManagedChannelBuilder delegateChannelBuilder, + ApiConfig apiConfig, + int poolSize, + GcpManagedChannelOptions options) { + this(delegateChannelBuilder, apiConfig, options); + if (poolSize != 0) { + this.maxSize = poolSize; + } + } + private void initOptions() { + GcpManagedChannelOptions.GcpChannelPoolOptions poolOptions = options.getChannelPoolOptions(); + if (poolOptions != null) { + this.maxSize = poolOptions.getMaxSize(); + this.maxConcurrentStreamsLowWatermark = poolOptions.getConcurrentStreamsLowWatermark(); + } initMetrics(); } diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelBuilder.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelBuilder.java index 061cc095..425cc52a 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelBuilder.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelBuilder.java @@ -63,7 +63,10 @@ public static GcpManagedChannelBuilder forDelegateBuilder(ManagedChannelBuilder< return new GcpManagedChannelBuilder(delegate); } - /** Sets the channel pool size. This will override the pool size configuration in ApiConfig. */ + /** Sets the maximum channel pool size. This will override the pool size configuration in ApiConfig. + * Deprecated. Use maxSize in GcpManagedChannelOptions.GcpChannelPoolOptions. + */ + @Deprecated public GcpManagedChannelBuilder setPoolSize(int poolSize) { this.poolSize = poolSize; return this; diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java index 8870bcb5..43cdfa15 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java @@ -29,19 +29,27 @@ public class GcpManagedChannelOptions { private static final Logger logger = Logger.getLogger(GcpManagedChannelOptions.class.getName()); + @Nullable private final GcpChannelPoolOptions channelPoolOptions; @Nullable private final GcpMetricsOptions metricsOptions; @Nullable private final GcpResiliencyOptions resiliencyOptions; public GcpManagedChannelOptions() { + channelPoolOptions = null; metricsOptions = null; resiliencyOptions = null; } public GcpManagedChannelOptions(Builder builder) { + channelPoolOptions = builder.channelPoolOptions; metricsOptions = builder.metricsOptions; resiliencyOptions = builder.resiliencyOptions; } + @Nullable + public GcpChannelPoolOptions getChannelPoolOptions() { + return channelPoolOptions; + } + @Nullable public GcpMetricsOptions getMetricsOptions() { return metricsOptions; @@ -63,12 +71,14 @@ public static Builder newBuilder(GcpManagedChannelOptions options) { } public static class Builder { + private GcpChannelPoolOptions channelPoolOptions; private GcpMetricsOptions metricsOptions; private GcpResiliencyOptions resiliencyOptions; public Builder() {} public Builder(GcpManagedChannelOptions options) { + this.channelPoolOptions = options.getChannelPoolOptions(); this.metricsOptions = options.getMetricsOptions(); this.resiliencyOptions = options.getResiliencyOptions(); } @@ -77,6 +87,17 @@ public GcpManagedChannelOptions build() { return new GcpManagedChannelOptions(this); } + /** + * Sets the channel pool configuration for the {@link GcpManagedChannel}. + * + * @param channelPoolOptions a {@link GcpChannelPoolOptions} to use as a channel pool + * configuration. + */ + public Builder withChannelPoolOptions(GcpChannelPoolOptions channelPoolOptions) { + this.channelPoolOptions = channelPoolOptions; + return this; + } + /** * Sets the metrics configuration for the {@link GcpManagedChannel}. * @@ -127,6 +148,82 @@ public Builder withResiliencyOptions(GcpResiliencyOptions resiliencyOptions) { } } + /** Channel pool configuration for the GCP managed channel. */ + public static class GcpChannelPoolOptions { + // The maximum number of channels in the pool. + private final int maxSize; + // If every channel in the pool has at least this amount of concurrent streams then a new channel will be created + // in the pool unless the pool reached its maximum size. + private final int concurrentStreamsLowWatermark; + + public GcpChannelPoolOptions(Builder builder) { + maxSize = builder.maxSize; + concurrentStreamsLowWatermark = builder.concurrentStreamsLowWatermark; + } + + public int getMaxSize() { + return maxSize; + } + + public int getConcurrentStreamsLowWatermark() { + return concurrentStreamsLowWatermark; + } + + /** Creates a new GcpChannelPoolOptions.Builder. */ + public static GcpChannelPoolOptions.Builder newBuilder() { + return new GcpChannelPoolOptions.Builder(); + } + + /** Creates a new GcpChannelPoolOptions.Builder from GcpChannelPoolOptions. */ + public static GcpChannelPoolOptions.Builder newBuilder(GcpChannelPoolOptions options) { + return new GcpChannelPoolOptions.Builder(options); + } + + public static class Builder { + private int maxSize = GcpManagedChannel.DEFAULT_MAX_CHANNEL; + private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM; + + public Builder() {} + + public Builder(GcpChannelPoolOptions options) { + this(); + if (options == null) { + return; + } + this.maxSize = options.getMaxSize(); + this.concurrentStreamsLowWatermark = options.getConcurrentStreamsLowWatermark(); + } + + public GcpChannelPoolOptions build() { + return new GcpChannelPoolOptions(this); + } + + /** + * Sets the maximum size of the channel pool. + * + * @param maxSize maximum number of channels the pool can have. + */ + public Builder setMaxSize(int maxSize) { + Preconditions.checkArgument(maxSize > 0, "Channel pool size must be positive."); + this.maxSize = maxSize; + return this; + } + + /** + * Sets the concurrent streams low watermark. + * If every channel in the pool has at least this amount of concurrent streams then a new + * channel will be created in the pool unless the pool reached its maximum size. + * + * @param concurrentStreamsLowWatermark number of streams every channel must reach before adding a new channel + * to the pool. + */ + public Builder setConcurrentStreamsLowWatermark(int concurrentStreamsLowWatermark) { + this.concurrentStreamsLowWatermark = concurrentStreamsLowWatermark; + return this; + } + } + } + /** Metrics configuration for the GCP managed channel. */ public static class GcpMetricsOptions { private final MetricRegistry metricRegistry; diff --git a/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto b/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto index 1301dd99..81987f19 100644 --- a/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto +++ b/grpc-gcp/src/main/proto/google/grpc/gcp/proto/grpc_gcp.proto @@ -21,14 +21,19 @@ option java_outer_classname = "GcpExtensionProto"; option java_package = "com.google.cloud.grpc.proto"; message ApiConfig { + // Deprecated. Use GcpManagedChannelOptions.GcpChannelPoolOptions class. // The channel pool configurations. - ChannelPoolConfig channel_pool = 2; + ChannelPoolConfig channel_pool = 2 [deprecated = true]; // The method configurations. repeated MethodConfig method = 1001; } + +// Deprecated. Use GcpManagedChannelOptions.GcpChannelPoolOptions class. message ChannelPoolConfig { + option deprecated = true; + // The max number of channels in the pool. uint32 max_size = 1; // The idle timeout (seconds) of channels without bound affinity sessions. diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 50c3b3c6..0635e538 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import com.google.cloud.grpc.GcpManagedChannel.ChannelRef; +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions; import com.google.cloud.grpc.MetricRegistryTestUtils.FakeMetricRegistry; @@ -130,6 +131,51 @@ public void testLoadApiConfigString() throws Exception { assertEquals(3, gcpChannel.methodToAffinity.size()); } + @Test + public void testUsesPoolOptions() { + resetGcpChannel(); + GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() + .setMaxSize(5) + .setConcurrentStreamsLowWatermark(50) + .build(); + GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(poolOptions) + .build(); + gcpChannel = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withOptions(options) + .build(); + assertEquals(0, gcpChannel.channelRefs.size()); + assertEquals(5, gcpChannel.getMaxSize()); + assertEquals(50, gcpChannel.getStreamsLowWatermark()); + } + + @Test + public void testPoolOptionsOverrideApiConfig() { + resetGcpChannel(); + final URL resource = GcpManagedChannelTest.class.getClassLoader().getResource(API_FILE); + assertNotNull(resource); + File configFile = new File(resource.getFile()); + GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() + .setMaxSize(5) + .setConcurrentStreamsLowWatermark(50) + .build(); + GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(poolOptions) + .build(); + gcpChannel = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withApiConfigJsonFile(configFile) + .withOptions(options) + .build(); + assertEquals(0, gcpChannel.channelRefs.size()); + assertEquals(5, gcpChannel.getMaxSize()); + assertEquals(50, gcpChannel.getStreamsLowWatermark()); + assertEquals(3, gcpChannel.methodToAffinity.size()); + } + @Test public void testGetChannelRefInitialization() { // Should not have a managedchannel by default. From fd7033c0cc9837e8734878c88e291e4f59de7eec Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 15 Dec 2021 15:57:10 -0800 Subject: [PATCH 24/87] Fix setGrpcServerAddress --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index aff34bff..2353f193 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -46,12 +46,12 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { GoogleCloudStorageOptions.builder() .setAppName("weiranf-app") .setGrpcEnabled(grpcEnabled) + .setGrpcServerAddress(Strings.isNullOrEmpty(args.host2) ? null : args.host2) .setStorageRootUrl("https://" + args.host) .setStorageServicePath(args.service_path) .setDirectPathPreferred(args.dp) .setReadChannelOptions( GoogleCloudStorageReadOptions.builder() - .setGrpcServerAddress(Strings.isNullOrEmpty(args.host2) ? null : args.host2) .setGrpcChecksumsEnabled(args.checksum) .build()) .setWriteChannelOptions( From c0736f536bb99a848f4c492eb09c5a392fbebfde Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 16 Dec 2021 12:00:16 -0800 Subject: [PATCH 25/87] Fix null error --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 754bbcb9..a9d6720c 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -42,11 +42,10 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { logger.warning("Please provide valid --access_token"); } - this.gcsOpts = + GoogleCloudStorageOptions.Builder optsBuilder = GoogleCloudStorageOptions.builder() .setAppName("weiranf-app") .setGrpcEnabled(grpcEnabled) - .setGrpcServerAddress(Strings.isNullOrEmpty(args.host2) ? null : args.host2) .setStorageRootUrl("https://" + args.host) .setStorageServicePath(args.service_path) .setTrafficDirectorEnabled(args.td) @@ -56,8 +55,11 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { .setGrpcChecksumsEnabled(args.checksum) .build()) .setWriteChannelOptions( - AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) - .build(); + AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()); + if (!Strings.isNullOrEmpty(args.host2)) { + optsBuilder.setGrpcServerAddress(args.host2); + } + this.gcsOpts = optsBuilder.build(); } public void startCalls(ResultTable results) throws InterruptedException, IOException { From d58cf86e1f93923fef31ce58a6da6fbe1f7b35b2 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 19 Jan 2022 14:17:47 -0500 Subject: [PATCH 26/87] doc: fix hostname in README for echo-client --- end2end-test-examples/echo-client/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/end2end-test-examples/echo-client/README.md b/end2end-test-examples/echo-client/README.md index 288386d7..ac97736d 100644 --- a/end2end-test-examples/echo-client/README.md +++ b/end2end-test-examples/echo-client/README.md @@ -8,25 +8,25 @@ Client sends out `numRpcs` number of unary requests to `host` sequentially with request payload size of `reqSize`, and expected response payload size of `rspSize`: ```sh -./gradlew run --args="--numRpcs=100 --reqSize=100 --resSize=100 --host=grpc-cloudapi1.googleapis.com" +./gradlew run --args="--numRpcs=100 --reqSize=100 --resSize=100 --host=grpc-cloudapi.googleapis.com" ``` Enable gRPC compression for both request and response with gzip: ```sh -./gradlew run --args="--numRpcs=100 --reqSize=100 --resSize=100 --reqComp=gzip --resComp=gzip --host=grpc-cloudapi1.googleapis.com" +./gradlew run --args="--numRpcs=100 --reqSize=100 --resSize=100 --reqComp=gzip --resComp=gzip --host=grpc-cloudapi.googleapis.com" ``` Sending requests infinitely with 10 seconds interval between requests. ```sh -./gradlew run --args="--numRpcs=0 --interval=10000 --reqSize=100 --resSize=100 --host=grpc-cloudapi1.googleapis.com" +./gradlew run --args="--numRpcs=0 --interval=10000 --reqSize=100 --resSize=100 --host=grpc-cloudapi.googleapis.com" ``` Receive server-streaming responses with 10 seconds interval. Re-create the stream after each 10 responses. ```sh -./gradlew run --args="--stream=true --numRpcs=10 --interval=10000 --host=grpc-cloudapi1.googleapis.com" +./gradlew run --args="--stream=true --numRpcs=10 --interval=10000 --host=grpc-cloudapi.googleapis.com" ``` Example results: From ed113f3073e35ed2160f467a447f21eb5a38188e Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 19 Jan 2022 15:09:29 -0800 Subject: [PATCH 27/87] Upgrade gRPC to 1.43 --- end2end-test-examples/gcs/build.gradle | 6 +++--- .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index b8772d02..7dfcb7e5 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -10,10 +10,10 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 def gcsioVersion = '2.2.5-SNAPSHOT' -def grpcVersion = '1.42.1' -def protobufVersion = '3.17.3' +def grpcVersion = '1.43.2' +def protobufVersion = '3.19.2' def protocVersion = protobufVersion -def conscryptVersion = '2.5.1' +def conscryptVersion = '2.5.2' repositories { mavenLocal() diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index f0d2e0a3..9df0ac2c 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -73,7 +73,8 @@ public GrpcClient(Args args) throws IOException { String target = args.host; if (args.td) { - target = "google-c2p:///" + target; + // TODO(veblush): Remove experimental suffix once this code is proven stable. + target = "google-c2p-experimental:///" + target; } ManagedChannelBuilder channelBuilder; From 7c920ced28ecc3179d9335cfc34611780c9269af Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 31 Jan 2022 13:40:44 -0800 Subject: [PATCH 28/87] Upgrade GCSIO to 2.2.5 --- end2end-test-examples/gcs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 7dfcb7e5..8c6d8984 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -9,7 +9,7 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 -def gcsioVersion = '2.2.5-SNAPSHOT' +def gcsioVersion = '2.2.5' def grpcVersion = '1.43.2' def protobufVersion = '3.19.2' def protocVersion = protobufVersion From 7f4b3bd15f0526e8d17d3fc47000e53285484d14 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 8 Dec 2021 14:51:48 -0800 Subject: [PATCH 29/87] Add optional round-robin channel selection for affinity binding calls. --- .../com/google/cloud/grpc/GcpClientCall.java | 7 +- .../google/cloud/grpc/GcpManagedChannel.java | 35 ++++ .../cloud/grpc/GcpManagedChannelOptions.java | 19 +++ .../cloud/grpc/SpannerIntegrationTest.java | 159 +++++++++++++++++- 4 files changed, 211 insertions(+), 9 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpClientCall.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpClientCall.java index 60c6c1a8..edd853cc 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpClientCall.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpClientCall.java @@ -115,7 +115,12 @@ public void sendMessage(ReqT message) { && delegateChannel.getChannelRef(keys.get(0)) != null) { key = keys.get(0); } - delegateChannelRef = delegateChannel.getChannelRef(key); + + if (affinity != null && affinity.getCommand().equals(AffinityConfig.Command.BIND)) { + delegateChannelRef = delegateChannel.getChannelRefForBind(); + } else { + delegateChannelRef = delegateChannel.getChannelRef(key); + } delegateChannelRef.activeStreamsCountIncr(); // Create the client call and do the previous operations. diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index b3b5095c..5f35f2aa 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -24,6 +24,7 @@ import com.google.cloud.grpc.proto.ApiConfig; import com.google.cloud.grpc.proto.MethodConfig; import com.google.common.annotations.VisibleForTesting; +import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.MessageOrBuilder; import io.grpc.CallOptions; @@ -62,6 +63,9 @@ public class GcpManagedChannel extends ManagedChannel { static final int DEFAULT_MAX_CHANNEL = 10; static final int DEFAULT_MAX_STREAM = 100; + @GuardedBy("this") + private Integer bindingIndex = -1; + private final ManagedChannelBuilder delegateChannelBuilder; private final GcpManagedChannelOptions options; private final boolean fallbackEnabled; @@ -772,6 +776,37 @@ public int getMaxActiveStreams() { return channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).max().orElse(0); } + /** + * Returns a {@link ChannelRef} from the pool for a binding call. + * If round-robin on bind is enabled, uses {@link #getChannelRefRoundRobin()} + * otherwise {@link #getChannelRef(String)} + * + * @return {@link ChannelRef} channel to use for a call. + */ + protected ChannelRef getChannelRefForBind() { + if (options.getChannelPoolOptions() != null && options.getChannelPoolOptions().isUseRoundRobinOnBind()) { + return getChannelRefRoundRobin(); + } + return getChannelRef(null); + } + + /** + * Returns a {@link ChannelRef} from the pool in round-robin manner. + * Creates a new channel in the pool until the pool reaches its max size. + * + * @return {@link ChannelRef} + */ + protected synchronized ChannelRef getChannelRefRoundRobin() { + if (channelRefs.size() < maxSize) { + return createNewChannel(); + } + bindingIndex++; + if (bindingIndex >= channelRefs.size()) { + bindingIndex = 0; + } + return channelRefs.get(bindingIndex); + } + /** * Pick a {@link ChannelRef} (and create a new one if necessary). If notReadyFallbackEnabled is * true in the {@link GcpResiliencyOptions} then instead of a channel in a non-READY state another diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java index 43cdfa15..9d5fe2d1 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java @@ -155,10 +155,13 @@ public static class GcpChannelPoolOptions { // If every channel in the pool has at least this amount of concurrent streams then a new channel will be created // in the pool unless the pool reached its maximum size. private final int concurrentStreamsLowWatermark; + // Use round-robin channel selection for affinity binding calls. + private final boolean useRoundRobinOnBind; public GcpChannelPoolOptions(Builder builder) { maxSize = builder.maxSize; concurrentStreamsLowWatermark = builder.concurrentStreamsLowWatermark; + useRoundRobinOnBind = builder.useRoundRobinOnBind; } public int getMaxSize() { @@ -179,9 +182,14 @@ public static GcpChannelPoolOptions.Builder newBuilder(GcpChannelPoolOptions opt return new GcpChannelPoolOptions.Builder(options); } + public boolean isUseRoundRobinOnBind() { + return useRoundRobinOnBind; + } + public static class Builder { private int maxSize = GcpManagedChannel.DEFAULT_MAX_CHANNEL; private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM; + private boolean useRoundRobinOnBind = false; public Builder() {} @@ -192,6 +200,7 @@ public Builder(GcpChannelPoolOptions options) { } this.maxSize = options.getMaxSize(); this.concurrentStreamsLowWatermark = options.getConcurrentStreamsLowWatermark(); + this.useRoundRobinOnBind = options.isUseRoundRobinOnBind(); } public GcpChannelPoolOptions build() { @@ -221,6 +230,16 @@ public Builder setConcurrentStreamsLowWatermark(int concurrentStreamsLowWatermar this.concurrentStreamsLowWatermark = concurrentStreamsLowWatermark; return this; } + + /** + * Enables/disables using round-robin channel selection for affinity binding calls. + * + * @param enabled If true, use round-robin channel selection for affinity binding calls. + */ + public Builder setUseRoundRobinOnBind(boolean enabled) { + this.useRoundRobinOnBind = enabled; + return this; + } } } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java index 2b657e45..8939ae13 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java @@ -110,6 +110,7 @@ public final class SpannerIntegrationTest { private static final ManagedChannelBuilder builder = ManagedChannelBuilder.forAddress(SPANNER_TARGET, 443); private GcpManagedChannel gcpChannel; + private GcpManagedChannel gcpChannelBRR; @BeforeClass public static void beforeClass() { @@ -242,10 +243,35 @@ private SpannerStub getSpannerStub() { /** A wrapper of checking the status of each channelRef in the gcpChannel. */ private void checkChannelRefs(int channels, int streams, int affinities) { - assertEquals(channels, gcpChannel.channelRefs.size()); + checkChannelRefs(gcpChannel, channels, streams, affinities); + } + + private void checkChannelRefs(GcpManagedChannel gcpChannel, int channels, int streams, int affinities) { + assertEquals("Channel pool size mismatch.", channels, gcpChannel.channelRefs.size()); for (int i = 0; i < channels; i++) { - assertEquals(streams, gcpChannel.channelRefs.get(i).getActiveStreamsCount()); - assertEquals(affinities, gcpChannel.channelRefs.get(i).getAffinityCount()); + assertEquals( + String.format("Channel %d streams mismatch.", i), + streams, gcpChannel.channelRefs.get(i).getActiveStreamsCount() + ); + assertEquals( + String.format("Channel %d affinities mismatch.", i), + affinities, + gcpChannel.channelRefs.get(i).getAffinityCount() + ); + } + } + + private void checkChannelRefs(int[] streams, int[] affinities) { + for (int i = 0; i < streams.length; i++) { + assertEquals( + String.format("Channel %d streams mismatch.", i), + streams[i], gcpChannel.channelRefs.get(i).getActiveStreamsCount() + ); + assertEquals( + String.format("Channel %d affinities mismatch.", i), + affinities[i], + gcpChannel.channelRefs.get(i).getAffinityCount() + ); } } @@ -292,10 +318,12 @@ private void deleteAsyncSessions(SpannerStub stub, List respNames) throw /** Helper Functions for FutureStub. */ private SpannerFutureStub getSpannerFutureStub() { + return getSpannerFutureStub(gcpChannel); + } + + private SpannerFutureStub getSpannerFutureStub(GcpManagedChannel gcpChannel) { GoogleCredentials creds = getCreds(); - SpannerFutureStub stub = - SpannerGrpc.newFutureStub(gcpChannel).withCallCredentials(MoreCallCredentials.from(creds)); - return stub; + return SpannerGrpc.newFutureStub(gcpChannel).withCallCredentials(MoreCallCredentials.from(creds)); } private List createFutureSessions(SpannerFutureStub stub) throws Exception { @@ -341,7 +369,7 @@ private void deleteFutureSessions(SpannerFutureStub stub, List futureNam @Rule public ExpectedException expectedEx = ExpectedException.none(); @Before - public void setupChannel() { + public void setupChannels() { File configFile = new File(SpannerIntegrationTest.class.getClassLoader().getResource(API_FILE).getFile()); gcpChannel = @@ -349,11 +377,25 @@ public void setupChannel() { GcpManagedChannelBuilder.forDelegateBuilder(builder) .withApiConfigJsonFile(configFile) .build(); + gcpChannelBRR = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withApiConfigJsonFile(configFile) + .withOptions(GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions( + GcpManagedChannelOptions.GcpChannelPoolOptions.newBuilder() + .setMaxSize(MAX_CHANNEL) + .setConcurrentStreamsLowWatermark(MAX_STREAM) + .setUseRoundRobinOnBind(true) + .build()) + .build()) + .build(); } @After - public void shutdownChannel() { + public void shutdownChannels() { gcpChannel.shutdownNow(); + gcpChannelBRR.shutdownNow(); } @Test @@ -403,6 +445,107 @@ public void testBatchCreateSessionsBlocking() throws Exception { checkChannelRefs(MAX_CHANNEL, 0, 0); } + @Test + public void testSessionsCreatedUsingRoundRobin() throws Exception { + SpannerFutureStub stub = getSpannerFutureStub(gcpChannelBRR); + List> futures = new ArrayList<>(); + assertEquals(ConnectivityState.IDLE, gcpChannelBRR.getState(false)); + + // Should create one session per channel. + CreateSessionRequest req = CreateSessionRequest.newBuilder().setDatabase(DATABASE_PATH).build(); + for (int i = 0; i < MAX_CHANNEL; i++) { + ListenableFuture future = stub.createSession(req); + futures.add(future); + } + // If round-robin in use as expected, then each channel should have 1 active stream with the CreateSession request. + checkChannelRefs(gcpChannelBRR, MAX_CHANNEL, 1, 0); + + // Collecting futures results. + String lastSession = ""; + for (ListenableFuture future : futures) { + lastSession = future.get().getName(); + } + // Since createSession will bind the key, check the number of keys bound with channels. + // Each channel should have 1 affinity key. + assertEquals(MAX_CHANNEL, gcpChannelBRR.affinityKeyToChannelRef.size()); + checkChannelRefs(gcpChannelBRR, MAX_CHANNEL, 0, 1); + + // Create a different request with the lastSession created. + ListenableFuture responseFuture = + stub.executeSql( + ExecuteSqlRequest.newBuilder() + .setSession(lastSession) + .setSql("select * FROM Users") + .build()); + // The ChannelRef which is bound with the lastSession. + GcpManagedChannel.ChannelRef currentChannel = + gcpChannelBRR.affinityKeyToChannelRef.get(lastSession); + // Verify the channel is in use. + assertEquals(1, currentChannel.getActiveStreamsCount()); + + // Create another 1 session per channel sequentially. + // Without the round-robin it won't use the currentChannel as it has more active streams (1) than other channels. + // But with round-robin each channel should get one create session request. + for (int i = 0; i < MAX_CHANNEL; i++) { + ListenableFuture future = stub.createSession(req); + future.get(); + } + ResultSet response = responseFuture.get(); + + // If round-robin in use, then each channel should now have 2 active stream with the CreateSession request. + checkChannelRefs(gcpChannelBRR, MAX_CHANNEL, 0, 2); + } + + @Test + public void testSessionsCreatedWithoutRoundRobin() throws Exception { + SpannerFutureStub stub = getSpannerFutureStub(); + List> futures = new ArrayList<>(); + assertEquals(ConnectivityState.IDLE, gcpChannel.getState(false)); + + // Should create one session per channel. + CreateSessionRequest req = CreateSessionRequest.newBuilder().setDatabase(DATABASE_PATH).build(); + for (int i = 0; i < MAX_CHANNEL; i++) { + ListenableFuture future = stub.createSession(req); + futures.add(future); + } + // Each channel should have 1 active stream with the CreateSession request because we create them concurrently. + checkChannelRefs(gcpChannel, MAX_CHANNEL, 1, 0); + + // Collecting futures results. + String lastSession = ""; + for (ListenableFuture future : futures) { + lastSession = future.get().getName(); + } + // Since createSession will bind the key, check the number of keys bound with channels. + // Each channel should have 1 affinity key. + assertEquals(MAX_CHANNEL, gcpChannel.affinityKeyToChannelRef.size()); + checkChannelRefs(MAX_CHANNEL, 0, 1); + + // Create a different request with the lastSession created. + ListenableFuture responseFuture = + stub.executeSql( + ExecuteSqlRequest.newBuilder() + .setSession(lastSession) + .setSql("select * FROM Users") + .build()); + // The ChannelRef which is bound with the lastSession. + GcpManagedChannel.ChannelRef currentChannel = + gcpChannel.affinityKeyToChannelRef.get(lastSession); + // Verify the channel is in use. + assertEquals(1, currentChannel.getActiveStreamsCount()); + + // Create another 1 session per channel sequentially. + // Without the round-robin it won't use the currentChannel as it has more active streams (1) than other channels. + for (int i = 0; i < MAX_CHANNEL; i++) { + ListenableFuture future = stub.createSession(req); + future.get(); + } + ResultSet response = responseFuture.get(); + + // Without round-robin the first channel will get all additional 3 sessions. + checkChannelRefs(new int[]{0, 0, 0}, new int[]{4, 1, 1}); + } + @Test public void testListSessionsFuture() throws Exception { SpannerFutureStub stub = getSpannerFutureStub(); From 1a25f4986b719c5c75b5558b6d8918b495d82e21 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 7 Feb 2022 17:53:56 -0800 Subject: [PATCH 30/87] Upgrade gRPC to 1.44 --- end2end-test-examples/gcs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 8c6d8984..4a07dd4b 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -10,7 +10,7 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 def gcsioVersion = '2.2.5' -def grpcVersion = '1.43.2' +def grpcVersion = '1.44.0' def protobufVersion = '3.19.2' def protocVersion = protobufVersion def conscryptVersion = '2.5.2' From 92a1846d9d314d5cb93ff8b292aace56312e8581 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 18 Feb 2022 16:07:55 -0800 Subject: [PATCH 31/87] TD channel creation --- .../src/main/java/io/grpc/gcs/GrpcClient.java | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 9df0ac2c..3fe0d86d 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -18,11 +18,13 @@ import com.google.storage.v2.WriteObjectRequest; import com.google.storage.v2.WriteObjectResponse; import com.google.storage.v2.WriteObjectSpec; +import io.grpc.Grpc; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.MethodDescriptor; import io.grpc.Status; import io.grpc.alts.ComputeEngineChannelBuilder; +import io.grpc.alts.GoogleDefaultChannelCredentials; import io.grpc.auth.MoreCallCredentials; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.StreamObserver; @@ -71,33 +73,28 @@ public GrpcClient(Args args) throws IOException { logger.warning("Please provide valid --access_token"); } - String target = args.host; + ManagedChannelBuilder channelBuilder; if (args.td) { // TODO(veblush): Remove experimental suffix once this code is proven stable. - target = "google-c2p-experimental:///" + target; - } - - ManagedChannelBuilder channelBuilder; - if (args.dp) { + String target = "google-c2p-experimental:///" + args.host; + channelBuilder = + Grpc.newChannelBuilder(target, GoogleDefaultChannelCredentials.newBuilder().build()); + } else if (args.dp) { ComputeEngineChannelBuilder gceChannelBuilder = - args.td - ? ComputeEngineChannelBuilder.forTarget(target) - : ComputeEngineChannelBuilder.forAddress(target, args.port); - - if (!args.td) { - String policy = args.rr ? "round_robin" : "pick_first"; - ImmutableMap policyStrategy = - ImmutableMap.of(policy, ImmutableMap.of()); - ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(policyStrategy)); - ImmutableMap grpcLbPolicy = - ImmutableMap.of("grpclb", childPolicy); - ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); - gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); - } + ComputeEngineChannelBuilder.forAddress(args.host, args.port); + + String policy = args.rr ? "round_robin" : "pick_first"; + ImmutableMap policyStrategy = + ImmutableMap.of(policy, ImmutableMap.of()); + ImmutableMap childPolicy = + ImmutableMap.of( + "childPolicy", ImmutableList.of(policyStrategy)); + ImmutableMap grpcLbPolicy = + ImmutableMap.of("grpclb", childPolicy); + ImmutableMap loadBalancingConfig = + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); if (args.flowControlWindow > 0) { Field delegateField = null; @@ -115,10 +112,7 @@ public GrpcClient(Args args) throws IOException { } channelBuilder = gceChannelBuilder; } else { - NettyChannelBuilder nettyChannelBuilder = - args.td - ? NettyChannelBuilder.forTarget(target) - : NettyChannelBuilder.forAddress(target, args.port); + NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(args.host, args.port); if (args.flowControlWindow > 0) { nettyChannelBuilder.flowControlWindow(args.flowControlWindow); } From 1ece5ac9fe138a484b984fcf95d124e5eeba13eb Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 18 Feb 2022 16:08:09 -0800 Subject: [PATCH 32/87] Add more logs for TD --- end2end-test-examples/gcs/logging.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/end2end-test-examples/gcs/logging.properties b/end2end-test-examples/gcs/logging.properties index 396eb81d..10fa920a 100644 --- a/end2end-test-examples/gcs/logging.properties +++ b/end2end-test-examples/gcs/logging.properties @@ -1,6 +1,8 @@ handlers = java.util.logging.ConsoleHandler java.util.logging.ConsoleHandler.level = ALL io.netty.handler.codec.http2.Http2FrameLogger.level = FINE +io.grpc.xds.XdsLogger.level = FINEST +io.grpc.ChannelLogger.level = FINEST .level = FINE io.grpc.netty.NettyClientHandler = ALL io.grpc.netty.NettyServerHandler = ALL From d7ac736053ef74a6702e30f02dae33a18c4ed620 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Tue, 29 Mar 2022 17:16:45 -0700 Subject: [PATCH 33/87] upgrade deps --- end2end-test-examples/echo-client/build.gradle | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 0dbcd791..4c470743 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -7,10 +7,11 @@ plugins { group 'io.grpc' version '2.2-SNAPSHOT' -sourceCompatibility = 1.8 +sourceCompatibility = 1.9 +targetCompatibility = 1.9 -def grpcVersion = '1.34.1' -def protobufVersion = '3.9.0' +def grpcVersion = '1.45.0' +def protobufVersion = '3.19.4' def protocVersion = protobufVersion repositories { @@ -23,7 +24,7 @@ dependencies { implementation "io.grpc:grpc-testing:${grpcVersion}" implementation "io.grpc:grpc-netty-shaded:${grpcVersion}" implementation "io.grpc:grpc-census:${grpcVersion}" - implementation "net.sourceforge.argparse4j:argparse4j:0.8.1" + implementation "net.sourceforge.argparse4j:argparse4j:0.9.0" implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" implementation "org.hdrhistogram:HdrHistogram:2.1.11" From c7fb40316301d683d43b40cd2943dce1861c0ef1 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Tue, 29 Mar 2022 17:17:06 -0700 Subject: [PATCH 34/87] WIP: sending metrics --- .../echo-client/build.gradle | 7 +- .../src/main/java/io/grpc/echo/Args.java | 10 ++ .../main/java/io/grpc/echo/EchoClient.java | 124 ++++++++++++++++-- .../grpc/echo/MetricsClientInterceptor.java | 45 +++++++ 4 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 4c470743..45fa8aaf 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -13,6 +13,7 @@ targetCompatibility = 1.9 def grpcVersion = '1.45.0' def protobufVersion = '3.19.4' def protocVersion = protobufVersion +def opencensusVersion = '0.31.0' repositories { mavenCentral() @@ -26,7 +27,11 @@ dependencies { implementation "io.grpc:grpc-census:${grpcVersion}" implementation "net.sourceforge.argparse4j:argparse4j:0.9.0" implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" - implementation "org.hdrhistogram:HdrHistogram:2.1.11" + implementation "org.hdrhistogram:HdrHistogram:2.1.12" + implementation "io.opencensus:opencensus-api:${opencensusVersion}" + implementation "io.opencensus:opencensus-impl:${opencensusVersion}" + implementation "io.opencensus:opencensus-exporter-metrics-ocagent:${opencensusVersion}" + implementation "io.opencensus:opencensus-exporter-stats-stackdriver:${opencensusVersion}" implementation "org.apache.commons:commons-math3:3.6.1" diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java index 623d4fc1..beb1173d 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java @@ -41,6 +41,9 @@ public class Args { final int logMaxSize; final int logMaxFiles; final boolean disableConsoleLog; + final String metricName; + final String metricTaskPrefix; + final String metricProbeName; Args(String[] args) throws ArgumentParserException { ArgumentParser parser = @@ -79,6 +82,9 @@ public class Args { parser.addArgument("--logMaxSize").type(Integer.class).setDefault(0); parser.addArgument("--logMaxFiles").type(Integer.class).setDefault(0); parser.addArgument("--disableConsoleLog").type(Boolean.class).setDefault(false); + parser.addArgument("--metricName").type(String.class).setDefault(""); + parser.addArgument("--metricTaskPrefix").type(String.class).setDefault(""); + parser.addArgument("--metricProbeName").type(String.class).setDefault(""); Namespace ns = parser.parseArgs(args); @@ -113,6 +119,10 @@ public class Args { logMaxSize = ns.getInt("logMaxSize"); logMaxFiles = ns.getInt("logMaxFiles"); disableConsoleLog = ns.getBoolean("disableConsoleLog"); + metricName = ns.getString("metricName"); + metricTaskPrefix = ns.getString("metricTaskPrefix"); + metricProbeName = ns.getString("metricProbeName"); + distrib = (qps > 0) ? new PoissonDistribution(1000/qps) : null; } } diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index e5429e27..0eb72fdd 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -1,12 +1,6 @@ package io.grpc.echo; -import io.grpc.Channel; -import io.grpc.ClientInterceptor; -import io.grpc.ClientInterceptors; -import io.grpc.ConnectivityState; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; +import io.grpc.*; import io.grpc.echo.Echo.EchoResponse; import io.grpc.echo.Echo.BatchEchoRequest; import io.grpc.echo.Echo.BatchEchoResponse; @@ -18,15 +12,20 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.grpc.stub.StreamObserver; + +import java.net.InetAddress; +import java.net.UnknownHostException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ssl.SSLException; + +import io.opencensus.metrics.*; import org.HdrHistogram.Histogram; public class EchoClient { @@ -44,9 +43,15 @@ public class EchoClient { private int rr; + private MetricRegistry metricRegistry; + private Map> errorCounts = new ConcurrentHashMap<>(); + final private String OTHER_STATUS = "OTHER"; + public EchoClient(Args args) throws SSLException { this.args = args; + setUpMetrics(); + channels = new ManagedChannel[args.numChannels]; asyncStubs = new GrpcCloudapiStub[args.numChannels]; rr = 0; @@ -79,6 +84,78 @@ public EchoClient(Args args) throws SSLException { } } + private void setUpMetrics() { + if (args.metricName.isEmpty()) { + return; + } + metricRegistry = Metrics.getMetricRegistry(); + + String hostname = "unknown"; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + logger.log(Level.WARNING, "Cannot get hostname", e); + } + + Map labels = new HashMap<>(); + labels.put( + LabelKey.create("prober_task", "Prober task identifier"), + LabelValue.create(args.metricTaskPrefix + ProcessHandle.current().pid() + "@" + hostname) + ); + if (!args.metricProbeName.isEmpty()) { + labels.put( + LabelKey.create("probe_name", "Prober name"), + LabelValue.create(args.metricProbeName) + ); + } + + final DerivedLongGauge presenceMetric = + metricRegistry.addDerivedLongGauge(args.metricName + "/presence", MetricOptions.builder() + .setDescription("Number of prober instances running") + .setUnit("1") + .setConstantLabels(labels) + .build()); + + final List errorKeys = new ArrayList<>(); + errorKeys.add(LabelKey.create("code", "The gRPC error code")); + errorKeys.add(LabelKey.create("sawGfe", "Whether Google load balancer response headers were present")); + + final DerivedLongCumulative errorsMetric = + metricRegistry.addDerivedLongCumulative(args.metricName + "/error-count", MetricOptions.builder() + .setDescription("Number of RPC errors") + .setUnit("1") + .setConstantLabels(labels) + .setLabelKeys(errorKeys) + .build()); + + final List emptyValues = new ArrayList<>(); + presenceMetric.removeTimeSeries(emptyValues); + presenceMetric.createTimeSeries(emptyValues, this, echoClient -> 1L); + + final List reportedStatuses = new ArrayList<>(); + reportedStatuses.add(Status.Code.DEADLINE_EXCEEDED.toString()); + reportedStatuses.add(Status.Code.UNAVAILABLE.toString()); + reportedStatuses.add(Status.Code.CANCELLED.toString()); + reportedStatuses.add(Status.Code.ABORTED.toString()); + reportedStatuses.add(Status.Code.INTERNAL.toString()); + reportedStatuses.add(OTHER_STATUS); + + for (String status : reportedStatuses) { + errorCounts.putIfAbsent(status, new ConcurrentHashMap<>()); + errorCounts.get(status).putIfAbsent(false, new AtomicLong()); + errorCounts.get(status).putIfAbsent(true, new AtomicLong()); + + for (boolean sawGfe : Arrays.asList(false, true)) { + final List errorValues = new ArrayList<>(); + errorValues.add(LabelValue.create(status)); + errorValues.add(LabelValue.create(String.valueOf(sawGfe))); + + errorsMetric.removeTimeSeries(errorValues); + errorsMetric.createTimeSeries(errorValues, this, echoClient -> echoClient.reportRpcErrors(status, sawGfe)); + } + } + } + private NettyChannelBuilder getChannelBuilder() throws SSLException { NettyChannelBuilder builder = NettyChannelBuilder.forTarget(args.host + ":" +args.port) .sslContext(GrpcSslContexts.forClient() @@ -113,6 +190,10 @@ private Channel createChannel(int i) throws SSLException { ClientInterceptor interceptor = new HeaderClientInterceptor(args); channel = ClientInterceptors.intercept(channel, interceptor); } + if (MetricsClientInterceptor.needsInterception(args)) { + ClientInterceptor interceptor = new MetricsClientInterceptor(this); + channel = ClientInterceptors.intercept(channel, interceptor); + } if (i == 0) { blockingChannelCreated = System.currentTimeMillis(); } @@ -260,4 +341,25 @@ public void echo(int id, CountDownLatch latch, Histogram histogram) throws SSLEx } //logger.info("Sync request: sent rpc#: " + rpcIndex); } + + private String statusToMetricLabel(Status status) { + switch (status.getCode()) { + case ABORTED: + case CANCELLED: + case DEADLINE_EXCEEDED: + case UNAVAILABLE: + case INTERNAL: + return status.getCode().toString(); + default: + return OTHER_STATUS; + } + } + + void registerRpcError(Status status, boolean sawGfe) { + errorCounts.get(statusToMetricLabel(status)).get(sawGfe).incrementAndGet(); + } + + private long reportRpcErrors(String status, boolean sawGfe) { + return errorCounts.get(status).get(sawGfe).get(); + } } diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java new file mode 100644 index 00000000..2e4e95c9 --- /dev/null +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java @@ -0,0 +1,45 @@ +package io.grpc.echo; + +import io.grpc.*; + +public class MetricsClientInterceptor implements ClientInterceptor { + final private EchoClient echoClient; + + public MetricsClientInterceptor(EchoClient echoClient) { + this.echoClient = echoClient; + } + + public static boolean needsInterception(Args args) { + return !args.metricName.isEmpty(); + } + + @Override + public ClientCall interceptCall(MethodDescriptor method, CallOptions callOptions, Channel next) { + return new ForwardingClientCall.SimpleForwardingClientCall<>(next.newCall(method, callOptions)) { + private boolean sawGfe = false; + + @Override + public void start(Listener responseListener, Metadata headers) { + headers.put(Metadata.Key.of("x-return-encrypted-headers", Metadata.ASCII_STRING_MARSHALLER), "true"); + super.start(new ForwardingClientCallListener.SimpleForwardingClientCallListener<>(responseListener) { + @Override + public void onHeaders(Metadata headers) { + if (!sawGfe) { + sawGfe = headers.containsKey(Metadata.Key.of("x-encrypted-debug-headers", Metadata.ASCII_STRING_MARSHALLER)); + } + super.onHeaders(headers); + } + + @Override + public void onClose(Status status, Metadata trailers) { + // Report status, saw GFE + if (!status.equals(Status.OK)) { + echoClient.registerRpcError(status, sawGfe); + } + super.onClose(status, trailers); + } + }, headers); + } + }; + } +} From 9cb7dcdf4ae2fd101fc01c895e6c057f6cbafab1 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Tue, 29 Mar 2022 17:33:57 -0700 Subject: [PATCH 35/87] WIP: add Stackdriver exporter --- .../src/main/java/io/grpc/echo/EchoClient.java | 18 ++++++++++++++++-- .../src/main/java/io/grpc/echo/TestMain.java | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index 0eb72fdd..6fe7fa80 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -13,6 +13,7 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.grpc.stub.StreamObserver; +import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.sql.Timestamp; @@ -25,6 +26,7 @@ import java.util.logging.Logger; import javax.net.ssl.SSLException; +import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; import io.opencensus.metrics.*; import org.HdrHistogram.Histogram; @@ -47,7 +49,7 @@ public class EchoClient { private Map> errorCounts = new ConcurrentHashMap<>(); final private String OTHER_STATUS = "OTHER"; - public EchoClient(Args args) throws SSLException { + public EchoClient(Args args) throws IOException { this.args = args; setUpMetrics(); @@ -84,7 +86,7 @@ public EchoClient(Args args) throws SSLException { } } - private void setUpMetrics() { + private void setUpMetrics() throws IOException { if (args.metricName.isEmpty()) { return; } @@ -154,6 +156,18 @@ private void setUpMetrics() { errorsMetric.createTimeSeries(errorValues, this, echoClient -> echoClient.reportRpcErrors(status, sawGfe)); } } + try { + // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring. + // Exporters use Application Default Credentials to authenticate. + // See https://developers.google.com/identity/protocols/application-default-credentials + // for more details. + // The minimum reporting period for Stackdriver is 1 minute. + StackdriverStatsExporter.createAndRegister(); + logger.log(Level.INFO, "Stackdriver metrics enabled!"); + } catch (IOException e) { + logger.log(Level.SEVERE, "StackdriverStatsExporter.createAndRegister()", e); + throw e; + } } private NettyChannelBuilder getChannelBuilder() throws SSLException { diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java index d7e7708b..f220a979 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java @@ -111,7 +111,7 @@ private static void runTest(Args args, EchoClient client) throws SSLException, I printResult(args, totalPayloadSize, totalRecvTime, histogram); } - private static void execTask(Args argObj) throws InterruptedException, SSLException { + private static void execTask(Args argObj) throws InterruptedException, IOException { EchoClient client = new EchoClient(argObj); try { logger.info("Start warm up..."); From 711fb607bf7ca82e206d42f71a301c9918ee5967 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 30 Mar 2022 10:56:12 -0700 Subject: [PATCH 36/87] downgrade grpc version --- end2end-test-examples/echo-client/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 45fa8aaf..7fdab56c 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -10,7 +10,7 @@ version '2.2-SNAPSHOT' sourceCompatibility = 1.9 targetCompatibility = 1.9 -def grpcVersion = '1.45.0' +def grpcVersion = '1.34.1' def protobufVersion = '3.19.4' def protocVersion = protobufVersion def opencensusVersion = '0.31.0' From 789350767931f4b328b410a38a40459ca4497f4f Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 30 Mar 2022 20:31:44 -0700 Subject: [PATCH 37/87] revert protobuf version --- end2end-test-examples/echo-client/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 7fdab56c..7db497fc 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -11,7 +11,7 @@ sourceCompatibility = 1.9 targetCompatibility = 1.9 def grpcVersion = '1.34.1' -def protobufVersion = '3.19.4' +def protobufVersion = '3.9.0' def protocVersion = protobufVersion def opencensusVersion = '0.31.0' From fa90affd1a3a18c8d6b56db67383e64066c07146 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 30 Mar 2022 20:32:18 -0700 Subject: [PATCH 38/87] add opencensus to pom.xml --- end2end-test-examples/echo-client/pom.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/end2end-test-examples/echo-client/pom.xml b/end2end-test-examples/echo-client/pom.xml index 22a579a2..0b41ad10 100644 --- a/end2end-test-examples/echo-client/pom.xml +++ b/end2end-test-examples/echo-client/pom.xml @@ -22,6 +22,7 @@ 1.34.1 3.9.0 ${protobuf.version} + 0.31.0 @@ -88,6 +89,26 @@ commons-math3 3.6.1 + + io.opencensus + opencensus-api + ${opencensus.version} + + + io.opencensus + opencensus-impl + ${opencensus.version} + + + io.opencensus + opencensus-exporter-metrics-ocagent + ${opencensus.version} + + + io.opencensus + opencensus-exporter-stats-stackdriver + ${opencensus.version} + junit junit From 47010ec5f35557f3c729a01490b862ac96f9b0f4 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 30 Mar 2022 20:53:23 -0700 Subject: [PATCH 39/87] bump version --- end2end-test-examples/echo-client/build.gradle | 2 +- end2end-test-examples/echo-client/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 7db497fc..4d8e264c 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -5,7 +5,7 @@ plugins { } group 'io.grpc' -version '2.2-SNAPSHOT' +version '2.3-SNAPSHOT' sourceCompatibility = 1.9 targetCompatibility = 1.9 diff --git a/end2end-test-examples/echo-client/pom.xml b/end2end-test-examples/echo-client/pom.xml index 0b41ad10..7e888029 100644 --- a/end2end-test-examples/echo-client/pom.xml +++ b/end2end-test-examples/echo-client/pom.xml @@ -6,7 +6,7 @@ 4.0.0 io.grpc echo-client - 2.2-SNAPSHOT + 2.3-SNAPSHOT 2019 From 1c072ac9d1db9920337d8a447bd386acf72a56f4 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 31 Mar 2022 09:57:56 -0700 Subject: [PATCH 40/87] add Dockerfile --- end2end-test-examples/echo-client/Dockerfile | 7 ++++++ end2end-test-examples/echo-client/README.md | 26 ++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 end2end-test-examples/echo-client/Dockerfile diff --git a/end2end-test-examples/echo-client/Dockerfile b/end2end-test-examples/echo-client/Dockerfile new file mode 100644 index 00000000..4c0c517f --- /dev/null +++ b/end2end-test-examples/echo-client/Dockerfile @@ -0,0 +1,7 @@ +FROM openjdk:11-jre + +ADD target/echo-client-2.3-SNAPSHOT.jar /app/ + +WORKDIR /app + +ENTRYPOINT ["java", "-jar", "echo-client-2.3-SNAPSHOT.jar"] diff --git a/end2end-test-examples/echo-client/README.md b/end2end-test-examples/echo-client/README.md index ac97736d..6597f3bd 100644 --- a/end2end-test-examples/echo-client/README.md +++ b/end2end-test-examples/echo-client/README.md @@ -84,3 +84,29 @@ Per sec Payload = 0.07 MB (exact amount of KB = 10000) `--logMaxFiles`: If log file size is limited rotate log files and keep this number of files. Default: unlimited. `--disableConsoleLog`: If logging to a file do not log to console. Default: false. + +## Deployment + +Build the jar using maven (`mvn clean package`) or docker: + +```shell +docker run -it --rm -v "$(pwd)":/usr/src/mymaven -w /usr/src/mymaven maven mvn clean package +``` + +Deploy the `target/echo-client-2.3-SNAPSHOT.jar` or build a docker image with the jar: + +```shell +docker build -t echo-client:2.3 . +``` + +Run the jar: + +```shell +java -jar echo-client-2.3-SNAPSHOT.jar --numRpcs=1 --reqSize=100 --resSize=100 --host=grpc-cloudapi.googleapis.com +``` + +Or run a container: + +```shell +docker run -it --rm echo-client:2.3 --numRpcs=1 --reqSize=100 --resSize=100 --host=grpc-cloudapi.googleapis.com +``` From cc75b6f5e17965be051fa094ac0849c5213922ee Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 31 Mar 2022 10:46:52 -0700 Subject: [PATCH 41/87] add metrics args to the readme --- end2end-test-examples/echo-client/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/end2end-test-examples/echo-client/README.md b/end2end-test-examples/echo-client/README.md index 6597f3bd..a0bb741c 100644 --- a/end2end-test-examples/echo-client/README.md +++ b/end2end-test-examples/echo-client/README.md @@ -85,6 +85,12 @@ Per sec Payload = 0.07 MB (exact amount of KB = 10000) `--disableConsoleLog`: If logging to a file do not log to console. Default: false. +`--metricName`: Ship metrics to Google Cloud Monitoring with this prefix for metric names. + +`--metricTaskPrefix`: Prefix for the process label for metrics. + +`--metricProbeName`: Additional label for metrics. + ## Deployment Build the jar using maven (`mvn clean package`) or docker: From efb38f62b302146f121596306f033d52a7d9f039 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 31 Mar 2022 14:39:36 -0700 Subject: [PATCH 42/87] remove ocagent dependency --- end2end-test-examples/echo-client/build.gradle | 1 - end2end-test-examples/echo-client/pom.xml | 5 ----- 2 files changed, 6 deletions(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 4d8e264c..b3172e23 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -30,7 +30,6 @@ dependencies { implementation "org.hdrhistogram:HdrHistogram:2.1.12" implementation "io.opencensus:opencensus-api:${opencensusVersion}" implementation "io.opencensus:opencensus-impl:${opencensusVersion}" - implementation "io.opencensus:opencensus-exporter-metrics-ocagent:${opencensusVersion}" implementation "io.opencensus:opencensus-exporter-stats-stackdriver:${opencensusVersion}" implementation "org.apache.commons:commons-math3:3.6.1" diff --git a/end2end-test-examples/echo-client/pom.xml b/end2end-test-examples/echo-client/pom.xml index 7e888029..48674798 100644 --- a/end2end-test-examples/echo-client/pom.xml +++ b/end2end-test-examples/echo-client/pom.xml @@ -99,11 +99,6 @@ opencensus-impl ${opencensus.version} - - io.opencensus - opencensus-exporter-metrics-ocagent - ${opencensus.version} - io.opencensus opencensus-exporter-stats-stackdriver From cb77761ec81cc2b9cde5302887fb27baa3920b31 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 6 Apr 2022 16:19:03 -0700 Subject: [PATCH 43/87] change Java target to 8 for echo client --- end2end-test-examples/echo-client/build.gradle | 4 ++-- end2end-test-examples/echo-client/pom.xml | 4 ++-- .../src/main/java/io/grpc/echo/MetricsClientInterceptor.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index b3172e23..5f0e226d 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -7,8 +7,8 @@ plugins { group 'io.grpc' version '2.3-SNAPSHOT' -sourceCompatibility = 1.9 -targetCompatibility = 1.9 +sourceCompatibility = 1.8 +targetCompatibility = 1.8 def grpcVersion = '1.34.1' def protobufVersion = '3.9.0' diff --git a/end2end-test-examples/echo-client/pom.xml b/end2end-test-examples/echo-client/pom.xml index 48674798..3e39ea3a 100644 --- a/end2end-test-examples/echo-client/pom.xml +++ b/end2end-test-examples/echo-client/pom.xml @@ -17,8 +17,8 @@ - 11 - 11 + 8 + 8 1.34.1 3.9.0 ${protobuf.version} diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java index 2e4e95c9..21b94e04 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/MetricsClientInterceptor.java @@ -15,13 +15,13 @@ public static boolean needsInterception(Args args) { @Override public ClientCall interceptCall(MethodDescriptor method, CallOptions callOptions, Channel next) { - return new ForwardingClientCall.SimpleForwardingClientCall<>(next.newCall(method, callOptions)) { + return new ForwardingClientCall.SimpleForwardingClientCall(next.newCall(method, callOptions)) { private boolean sawGfe = false; @Override public void start(Listener responseListener, Metadata headers) { headers.put(Metadata.Key.of("x-return-encrypted-headers", Metadata.ASCII_STRING_MARSHALLER), "true"); - super.start(new ForwardingClientCallListener.SimpleForwardingClientCallListener<>(responseListener) { + super.start(new ForwardingClientCallListener.SimpleForwardingClientCallListener(responseListener) { @Override public void onHeaders(Metadata headers) { if (!sawGfe) { From 85f9a94f1540896df5c6305a873c435a17872730 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 7 Apr 2022 10:48:02 -0700 Subject: [PATCH 44/87] change getting a pid for echo-client metrics --- .../echo-client/src/main/java/io/grpc/echo/EchoClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index 6fe7fa80..0c5f3ff8 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -13,6 +13,7 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.grpc.stub.StreamObserver; +import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; @@ -99,10 +100,12 @@ private void setUpMetrics() throws IOException { logger.log(Level.WARNING, "Cannot get hostname", e); } + final String pid = new File("/proc/self").getCanonicalFile().getName(); + Map labels = new HashMap<>(); labels.put( LabelKey.create("prober_task", "Prober task identifier"), - LabelValue.create(args.metricTaskPrefix + ProcessHandle.current().pid() + "@" + hostname) + LabelValue.create(args.metricTaskPrefix + pid + "@" + hostname) ); if (!args.metricProbeName.isEmpty()) { labels.put( From 677d863d694a3fa72dabf8921e1a19b27c05ff8e Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Sun, 10 Apr 2022 13:22:05 -0700 Subject: [PATCH 45/87] echo-client: force global resource for stackdriver --- .../echo-client/src/main/java/io/grpc/echo/EchoClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index 0c5f3ff8..658a174f 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -1,5 +1,6 @@ package io.grpc.echo; +import com.google.api.MonitoredResource; import io.grpc.*; import io.grpc.echo.Echo.EchoResponse; import io.grpc.echo.Echo.BatchEchoRequest; @@ -27,6 +28,7 @@ import java.util.logging.Logger; import javax.net.ssl.SSLException; +import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; import io.opencensus.metrics.*; import org.HdrHistogram.Histogram; @@ -165,7 +167,9 @@ private void setUpMetrics() throws IOException { // See https://developers.google.com/identity/protocols/application-default-credentials // for more details. // The minimum reporting period for Stackdriver is 1 minute. - StackdriverStatsExporter.createAndRegister(); + StackdriverStatsExporter.createAndRegister(StackdriverStatsConfiguration.builder() + .setMonitoredResource(MonitoredResource.newBuilder().setType("global").build()) + .build()); logger.log(Level.INFO, "Stackdriver metrics enabled!"); } catch (IOException e) { logger.log(Level.SEVERE, "StackdriverStatsExporter.createAndRegister()", e); From 564cb1101eca3a207cb96943fc8c554143139d88 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 13 Apr 2022 14:37:35 -0700 Subject: [PATCH 46/87] Added warms parameter --- .../gcs/src/main/java/io/grpc/gcs/Args.java | 3 +++ .../gcs/src/main/java/io/grpc/gcs/ResultTable.java | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java index 7d9099d6..af0bd4af 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java @@ -19,6 +19,7 @@ public class Args { private static final int PORT = 443; final int calls; + final int warmups; final String cookie; final String host; final String host2; @@ -51,6 +52,7 @@ public class Args { .description("GCS client java binary"); parser.addArgument("--calls").type(Integer.class).setDefault(1); + parser.addArgument("--warmups").type(Integer.class).setDefault(0); parser.addArgument("--cookie").type(String.class).setDefault(""); parser.addArgument("--host").type(String.class).setDefault(DEFAULT_HOST); parser.addArgument("--host2").type(String.class).setDefault(""); @@ -80,6 +82,7 @@ public class Args { // Read args calls = ns.getInt("calls"); + warmups = ns.getInt("warmups"); cookie = ns.getString("cookie"); host = ns.getString("host"); host2 = ns.getString("host2"); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java index ec7a1928..fe769d6b 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java @@ -11,10 +11,12 @@ public class ResultTable { private Args args; private Long startTime; private Long endTime; + private int warmupCount; private List results; public ResultTable(Args args) { this.args = args; + this.warmupCount = args.warmups * args.threads; this.results = new ArrayList<>(); } @@ -37,7 +39,9 @@ public void reportResult(long duration) { ord = results.size(); } if (this.args.verboseResult) { - System.out.format("### Result: ord=%d elapsed=%d\n", ord, duration); + System.out.format( + "### Result: ord=%d elapsed=%d%s\n", + ord, duration, results.size() <= warmupCount ? " [WARM-UP]" : ""); System.out.flush(); } } @@ -49,8 +53,11 @@ private static class BenchmarkResult { public void printResult() throws IOException { synchronized (this) { + results.subList(0, Math.min(results.size(), warmupCount)).clear(); + if (results.size() == 0) return; Collections.sort(results); + int n = results.size(); double totalSeconds = 0; long totalDur = endTime - startTime; From 12b600f21e9484ffea692445e4bcef2d0ea520d5 Mon Sep 17 00:00:00 2001 From: Ortal Majlin Date: Wed, 20 Apr 2022 20:25:12 +0000 Subject: [PATCH 47/87] echo-client: edit streamingEcho API - 1. Set MessageCount according to new arg numMsgs 2. Set MessageInterval according to new arg msgsInterval 3. Add response_size to StreamEchoRequest proto request, and set according to resSize --- .../src/main/java/io/grpc/echo/Args.java | 8 +- .../main/java/io/grpc/echo/EchoClient.java | 190 ++++++++++-------- .../src/main/java/io/grpc/echo/TestMain.java | 2 +- .../echo-client/src/main/proto/echo.proto | 8 +- 4 files changed, 116 insertions(+), 92 deletions(-) diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java index beb1173d..a57bfe5c 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/Args.java @@ -44,6 +44,8 @@ public class Args { final String metricName; final String metricTaskPrefix; final String metricProbeName; + final int numMsgs; + final int msgsInterval; Args(String[] args) throws ArgumentParserException { ArgumentParser parser = @@ -85,6 +87,8 @@ public class Args { parser.addArgument("--metricName").type(String.class).setDefault(""); parser.addArgument("--metricTaskPrefix").type(String.class).setDefault(""); parser.addArgument("--metricProbeName").type(String.class).setDefault(""); + parser.addArgument("--numMsgs").type(Integer.class).setDefault(1); + parser.addArgument("--msgsInterval").type(Integer.class).setDefault(0); Namespace ns = parser.parseArgs(args); @@ -122,7 +126,9 @@ public class Args { metricName = ns.getString("metricName"); metricTaskPrefix = ns.getString("metricTaskPrefix"); metricProbeName = ns.getString("metricProbeName"); + numMsgs = ns.getInt("numMsgs"); + msgsInterval = ns.getInt("msgsInterval"); - distrib = (qps > 0) ? new PoissonDistribution(1000/qps) : null; + distrib = (qps > 0) ? new PoissonDistribution(1000 / qps) : null; } } diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index 658a174f..c092625d 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -2,9 +2,9 @@ import com.google.api.MonitoredResource; import io.grpc.*; -import io.grpc.echo.Echo.EchoResponse; import io.grpc.echo.Echo.BatchEchoRequest; import io.grpc.echo.Echo.BatchEchoResponse; +import io.grpc.echo.Echo.EchoResponse; import io.grpc.echo.Echo.EchoWithResponseSizeRequest; import io.grpc.echo.Echo.StreamEchoRequest; import io.grpc.echo.GrpcCloudapiGrpc.GrpcCloudapiBlockingStub; @@ -13,7 +13,9 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.grpc.stub.StreamObserver; - +import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; +import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; +import io.opencensus.metrics.*; import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -27,10 +29,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ssl.SSLException; - -import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; -import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; -import io.opencensus.metrics.*; import org.HdrHistogram.Histogram; public class EchoClient { @@ -50,7 +48,7 @@ public class EchoClient { private MetricRegistry metricRegistry; private Map> errorCounts = new ConcurrentHashMap<>(); - final private String OTHER_STATUS = "OTHER"; + private final String OTHER_STATUS = "OTHER"; public EchoClient(Args args) throws IOException { this.args = args; @@ -106,34 +104,36 @@ private void setUpMetrics() throws IOException { Map labels = new HashMap<>(); labels.put( - LabelKey.create("prober_task", "Prober task identifier"), - LabelValue.create(args.metricTaskPrefix + pid + "@" + hostname) - ); + LabelKey.create("prober_task", "Prober task identifier"), + LabelValue.create(args.metricTaskPrefix + pid + "@" + hostname)); if (!args.metricProbeName.isEmpty()) { labels.put( - LabelKey.create("probe_name", "Prober name"), - LabelValue.create(args.metricProbeName) - ); + LabelKey.create("probe_name", "Prober name"), LabelValue.create(args.metricProbeName)); } final DerivedLongGauge presenceMetric = - metricRegistry.addDerivedLongGauge(args.metricName + "/presence", MetricOptions.builder() - .setDescription("Number of prober instances running") - .setUnit("1") - .setConstantLabels(labels) - .build()); + metricRegistry.addDerivedLongGauge( + args.metricName + "/presence", + MetricOptions.builder() + .setDescription("Number of prober instances running") + .setUnit("1") + .setConstantLabels(labels) + .build()); final List errorKeys = new ArrayList<>(); errorKeys.add(LabelKey.create("code", "The gRPC error code")); - errorKeys.add(LabelKey.create("sawGfe", "Whether Google load balancer response headers were present")); + errorKeys.add( + LabelKey.create("sawGfe", "Whether Google load balancer response headers were present")); final DerivedLongCumulative errorsMetric = - metricRegistry.addDerivedLongCumulative(args.metricName + "/error-count", MetricOptions.builder() - .setDescription("Number of RPC errors") - .setUnit("1") - .setConstantLabels(labels) - .setLabelKeys(errorKeys) - .build()); + metricRegistry.addDerivedLongCumulative( + args.metricName + "/error-count", + MetricOptions.builder() + .setDescription("Number of RPC errors") + .setUnit("1") + .setConstantLabels(labels) + .setLabelKeys(errorKeys) + .build()); final List emptyValues = new ArrayList<>(); presenceMetric.removeTimeSeries(emptyValues); @@ -158,7 +158,8 @@ private void setUpMetrics() throws IOException { errorValues.add(LabelValue.create(String.valueOf(sawGfe))); errorsMetric.removeTimeSeries(errorValues); - errorsMetric.createTimeSeries(errorValues, this, echoClient -> echoClient.reportRpcErrors(status, sawGfe)); + errorsMetric.createTimeSeries( + errorValues, this, echoClient -> echoClient.reportRpcErrors(status, sawGfe)); } } try { @@ -167,8 +168,9 @@ private void setUpMetrics() throws IOException { // See https://developers.google.com/identity/protocols/application-default-credentials // for more details. // The minimum reporting period for Stackdriver is 1 minute. - StackdriverStatsExporter.createAndRegister(StackdriverStatsConfiguration.builder() - .setMonitoredResource(MonitoredResource.newBuilder().setType("global").build()) + StackdriverStatsExporter.createAndRegister( + StackdriverStatsConfiguration.builder() + .setMonitoredResource(MonitoredResource.newBuilder().setType("global").build()) .build()); logger.log(Level.INFO, "Stackdriver metrics enabled!"); } catch (IOException e) { @@ -178,10 +180,12 @@ private void setUpMetrics() throws IOException { } private NettyChannelBuilder getChannelBuilder() throws SSLException { - NettyChannelBuilder builder = NettyChannelBuilder.forTarget(args.host + ":" +args.port) - .sslContext(GrpcSslContexts.forClient() - .trustManager(InsecureTrustManagerFactory.INSTANCE) - .build()); + NettyChannelBuilder builder = + NettyChannelBuilder.forTarget(args.host + ":" + args.port) + .sslContext( + GrpcSslContexts.forClient() + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .build()); if (!args.overrideService.isEmpty()) { builder.overrideAuthority(args.overrideService); } @@ -193,12 +197,16 @@ private NettyChannelBuilder getChannelBuilder() throws SSLException { return builder; } - private static void watchStateChange(ManagedChannel channel, ConnectivityState currentState, int i) { - channel.notifyWhenStateChanged(currentState, () -> { - ConnectivityState newState = channel.getState(false); - logger.fine(String.format("Channel %d state changed: %s -> %s", i, currentState, newState)); - watchStateChange(channel, newState, i); - }); + private static void watchStateChange( + ManagedChannel channel, ConnectivityState currentState, int i) { + channel.notifyWhenStateChanged( + currentState, + () -> { + ConnectivityState newState = channel.getState(false); + logger.fine( + String.format("Channel %d state changed: %s -> %s", i, currentState, newState)); + watchStateChange(channel, newState, i); + }); } private Channel createChannel(int i) throws SSLException { @@ -244,38 +252,44 @@ private GrpcCloudapiStub getNextAsyncStub() { return next; } - public void asyncEcho(int id, CountDownLatch latch, Histogram histogram) { - EchoWithResponseSizeRequest request = EchoWithResponseSizeRequest.newBuilder() - .setEchoMsg(generatePayload(args.reqSize * 1024)) - .setResponseSize(args.resSize) - .build(); + EchoWithResponseSizeRequest request = + EchoWithResponseSizeRequest.newBuilder() + .setEchoMsg(generatePayload(args.reqSize * 1024)) + .setResponseSize(args.resSize) + .build(); GrpcCloudapiStub stub = getNextAsyncStub(); - stub.withDeadlineAfter(args.timeout, TimeUnit.MILLISECONDS).echoWithResponseSize( - request, - new StreamObserver() { - long start = System.currentTimeMillis(); - - @Override - public void onNext(EchoResponse value) {} - - @Override - public void onError(Throwable t) { - if (latch != null) latch.countDown(); - Status status = Status.fromThrowable(t); - long elapsed = System.currentTimeMillis() - start; - logger.warning(String.format("Encountered an error in %dth echo RPC (startTime: %s, elapsed: %dms). Status: %s", id, new Timestamp(start), elapsed, status)); - t.printStackTrace(); - } - - @Override - public void onCompleted() { - long now = System.currentTimeMillis(); - if (histogram != null) histogram.recordValue(now - start); - if (latch != null) latch.countDown(); - //logger.info(String.format("%dth echo RPC succeeded. Start time: %s. Requests left: %d", id, new Timestamp(start), latch.getCount())); - } - }); + stub.withDeadlineAfter(args.timeout, TimeUnit.MILLISECONDS) + .echoWithResponseSize( + request, + new StreamObserver() { + long start = System.currentTimeMillis(); + + @Override + public void onNext(EchoResponse value) {} + + @Override + public void onError(Throwable t) { + if (latch != null) latch.countDown(); + Status status = Status.fromThrowable(t); + long elapsed = System.currentTimeMillis() - start; + logger.warning( + String.format( + "Encountered an error in %dth echo RPC (startTime: %s, elapsed: %dms)." + + " Status: %s", + id, new Timestamp(start), elapsed, status)); + t.printStackTrace(); + } + + @Override + public void onCompleted() { + long now = System.currentTimeMillis(); + if (histogram != null) histogram.recordValue(now - start); + if (latch != null) latch.countDown(); + // logger.info(String.format("%dth echo RPC succeeded. Start time: %s. Requests + // left: %d", id, new Timestamp(start), latch.getCount())); + } + }); } private String generatePayload(int numBytes) { @@ -289,10 +303,12 @@ private String generatePayload(int numBytes) { void streamingEcho() { long start = 0; try { - StreamEchoRequest request = StreamEchoRequest.newBuilder() - .setMessageCount(args.numRpcs) - .setMessageInterval(Math.max(args.interval, STREAMING_MIN_INTERVAL)) - .build(); + StreamEchoRequest request = + StreamEchoRequest.newBuilder() + .setMessageCount(args.numMsgs) + .setMessageInterval(args.msgsInterval) + .setResponseSizePerMsg(args.resSize) + .build(); start = System.currentTimeMillis(); Iterator iter = blockingStub.echoStream(request); for (long counter = 1; iter.hasNext(); ++counter) { @@ -311,26 +327,28 @@ void blockingEcho(Histogram histogram) throws SSLException { long start = 0; try { if (args.resType == 0) { - EchoWithResponseSizeRequest request = EchoWithResponseSizeRequest.newBuilder() - .setEchoMsg(generatePayload(args.reqSize * 1024)) - .setResponseSize(args.resSize) - .build(); + EchoWithResponseSizeRequest request = + EchoWithResponseSizeRequest.newBuilder() + .setEchoMsg(generatePayload(args.reqSize * 1024)) + .setResponseSize(args.resSize) + .build(); start = System.currentTimeMillis(); - if (args.recreateChannelSeconds >= 0 && blockingChannelCreated < start - args.recreateChannelSeconds * 1000) { + if (args.recreateChannelSeconds >= 0 + && blockingChannelCreated < start - args.recreateChannelSeconds * 1000) { reCreateBlockingStub(); } blockingStub .withDeadlineAfter(args.timeout, TimeUnit.MILLISECONDS) .echoWithResponseSize(request); } else { - BatchEchoRequest request = BatchEchoRequest.newBuilder() - .setEchoMsg(generatePayload(args.reqSize * 1024)) - .setResponseType(args.resType) - .build(); + BatchEchoRequest request = + BatchEchoRequest.newBuilder() + .setEchoMsg(generatePayload(args.reqSize * 1024)) + .setResponseType(args.resType) + .build(); start = System.currentTimeMillis(); - BatchEchoResponse response = blockingStub - .withDeadlineAfter(args.timeout, TimeUnit.MILLISECONDS) - .batchEcho(request); + BatchEchoResponse response = + blockingStub.withDeadlineAfter(args.timeout, TimeUnit.MILLISECONDS).batchEcho(request); List sizeList = new ArrayList<>(); for (EchoResponse r : response.getEchoResponsesList()) { sizeList.add(r.getSerializedSize()); @@ -356,11 +374,11 @@ public void echo(int id, CountDownLatch latch, Histogram histogram) throws SSLEx } if (args.async) { asyncEcho(id, latch, histogram); - //logger.info("Async request: sent rpc#: " + rpcIndex); + // logger.info("Async request: sent rpc#: " + rpcIndex); } else { blockingEcho(histogram); } - //logger.info("Sync request: sent rpc#: " + rpcIndex); + // logger.info("Sync request: sent rpc#: " + rpcIndex); } private String statusToMetricLabel(Status status) { diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java index f220a979..6c7681b4 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/TestMain.java @@ -76,7 +76,7 @@ private static void runTest(Args args, EchoClient client) throws SSLException, I long totalPayloadSize = 0; long startFirst = System.currentTimeMillis(); - for (int i = 0; args.stream || rpcsToDo == 0 || i < rpcsToDo; i++) { + for (int i = 0; rpcsToDo == 0 || i < rpcsToDo; i++) { if (args.async) { if (args.distrib != null) { int sample = args.distrib.sample(); diff --git a/end2end-test-examples/echo-client/src/main/proto/echo.proto b/end2end-test-examples/echo-client/src/main/proto/echo.proto index b9ab888d..6695fbc0 100644 --- a/end2end-test-examples/echo-client/src/main/proto/echo.proto +++ b/end2end-test-examples/echo-client/src/main/proto/echo.proto @@ -2,10 +2,10 @@ syntax = "proto3"; package e2e_service; -option java_package = "io.grpc.echo"; - import "google/api/annotations.proto"; +option java_package = "io.grpc.echo"; + // Request message type for simple echo. message EchoRequest { string string_to_echo = 1; @@ -33,6 +33,7 @@ message EchoWithResponseSizeRequest { message StreamEchoRequest { int32 message_count = 1; int32 message_interval = 2; + int32 response_size_per_msg = 3; } // A simple service to test and debug in an E2E environment @@ -55,7 +56,7 @@ service GrpcCloudapi { // A simple stream endpoint rpc EchoStream(StreamEchoRequest) returns (stream EchoResponse) { option (google.api.http) = { - get: "/v1/stream/{message_count}/{message_interval}" + get: "/v1/stream/{message_count}/{message_interval}/{response_size_per_msg}" }; } @@ -65,5 +66,4 @@ service GrpcCloudapi { get: "/v1/batch/{echo_msg}/{response_type}" }; } - } From 316c705a7f8e4438d22ef7f1fb395686be79e6ce Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 27 Apr 2022 19:02:08 -0400 Subject: [PATCH 48/87] feat: enable grpc client metrics --- end2end-test-examples/echo-client/build.gradle | 1 + end2end-test-examples/echo-client/pom.xml | 5 +++++ .../echo-client/src/main/java/io/grpc/echo/EchoClient.java | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/end2end-test-examples/echo-client/build.gradle b/end2end-test-examples/echo-client/build.gradle index 5f0e226d..cd6b5475 100644 --- a/end2end-test-examples/echo-client/build.gradle +++ b/end2end-test-examples/echo-client/build.gradle @@ -31,6 +31,7 @@ dependencies { implementation "io.opencensus:opencensus-api:${opencensusVersion}" implementation "io.opencensus:opencensus-impl:${opencensusVersion}" implementation "io.opencensus:opencensus-exporter-stats-stackdriver:${opencensusVersion}" + implementation 'io.opencensus:opencensus-contrib-grpc-metrics:${opencensus.version}' implementation "org.apache.commons:commons-math3:3.6.1" diff --git a/end2end-test-examples/echo-client/pom.xml b/end2end-test-examples/echo-client/pom.xml index 3e39ea3a..3e921199 100644 --- a/end2end-test-examples/echo-client/pom.xml +++ b/end2end-test-examples/echo-client/pom.xml @@ -104,6 +104,11 @@ opencensus-exporter-stats-stackdriver ${opencensus.version} + + io.opencensus + opencensus-contrib-grpc-metrics + ${opencensus.version} + junit junit diff --git a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java index c092625d..25f13f99 100644 --- a/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java +++ b/end2end-test-examples/echo-client/src/main/java/io/grpc/echo/EchoClient.java @@ -13,6 +13,7 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.grpc.stub.StreamObserver; +import io.opencensus.contrib.grpc.metrics.RpcViews; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; import io.opencensus.metrics.*; @@ -88,6 +89,9 @@ public EchoClient(Args args) throws IOException { } private void setUpMetrics() throws IOException { + // Configure standard gRPC client metrics + RpcViews.registerClientGrpcViews(); + if (args.metricName.isEmpty()) { return; } From 14523f53849f0169d59fae3e7f6e3382ee6e14e8 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 28 Apr 2022 13:58:05 -0700 Subject: [PATCH 49/87] Compute qps correctly --- .../main/java/io/grpc/gcs/ResultTable.java | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java index fe769d6b..3a133f88 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java @@ -5,14 +5,21 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; public class ResultTable { private Args args; - private Long startTime; - private Long endTime; private int warmupCount; - private List results; + private long startTime; + private long endTime; + + private static class Result { + public long startTime; + public long duration; + } + + private List results; public ResultTable(Args args) { this.args = args; @@ -35,7 +42,10 @@ public void stop() { public void reportResult(long duration) { int ord; synchronized (this) { - results.add(duration); + Result result = new Result(); + result.startTime = System.currentTimeMillis() - duration; + result.duration = duration; + results.add(result); ord = results.size(); } if (this.args.verboseResult) { @@ -56,23 +66,29 @@ public void printResult() throws IOException { results.subList(0, Math.min(results.size(), warmupCount)).clear(); if (results.size() == 0) return; - Collections.sort(results); int n = results.size(); - double totalSeconds = 0; - long totalDur = endTime - startTime; - for (Long ms : results) { - totalSeconds += ms / 1000.0; - } + long totalDur = + results.get(n - 1).startTime + results.get(n - 1).duration - results.get(0).startTime; + double totalSec = totalDur / 1000.0; + + Collections.sort( + results, + new Comparator() { + @Override + public int compare(Result o1, Result o2) { + return Long.compare(o1.duration, o2.duration); + } + }); Gson gson = new Gson(); BenchmarkResult benchmarkResult = new BenchmarkResult(); - benchmarkResult.min = results.get(0); - benchmarkResult.p50 = results.get((int) (n * 0.05)); - benchmarkResult.p90 = results.get((int) (n * 0.90)); - benchmarkResult.p99 = results.get((int) (n * 0.99)); - benchmarkResult.p999 = results.get((int) (n * 0.999)); - benchmarkResult.qps = n / totalSeconds; + benchmarkResult.min = results.get(0).duration; + benchmarkResult.p50 = results.get((int) (n * 0.05)).duration; + benchmarkResult.p90 = results.get((int) (n * 0.90)).duration; + benchmarkResult.p99 = results.get((int) (n * 0.99)).duration; + benchmarkResult.p999 = results.get((int) (n * 0.999)).duration; + benchmarkResult.qps = n / totalSec; if (!args.latencyFilename.isEmpty()) { FileWriter writer = new FileWriter(args.latencyFilename); gson.toJson(benchmarkResult, writer); @@ -81,7 +97,7 @@ public void printResult() throws IOException { System.out.println( String.format( "****** Test Results [client: %s, method: %s, size: %d, threads: %d, dp: %s, calls:" - + " %d]: \n" + + " %d, qps: %f]: \n" + "\t\tMin\tp5\tp10\tp25\tp50\tp75\tp90\tp99\tMax\tTotal\n" + " Time(ms)\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", args.client, @@ -90,15 +106,16 @@ public void printResult() throws IOException { args.threads, args.dp, n, - results.get(0), - results.get((int) (n * 0.05)), - results.get((int) (n * 0.1)), - results.get((int) (n * 0.25)), - results.get((int) (n * 0.50)), - results.get((int) (n * 0.75)), - results.get((int) (n * 0.90)), - results.get((int) (n * 0.99)), - results.get(n - 1), + n / totalSec, + results.get(0).duration, + results.get((int) (n * 0.05)).duration, + results.get((int) (n * 0.1)).duration, + results.get((int) (n * 0.25)).duration, + results.get((int) (n * 0.50)).duration, + results.get((int) (n * 0.75)).duration, + results.get((int) (n * 0.90)).duration, + results.get((int) (n * 0.99)).duration, + results.get(n - 1).duration, totalDur)); } } From 85be6baeade8a1dd7daaeb5ff2bbfbb2ec47ae8c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 15:37:57 -0700 Subject: [PATCH 50/87] Bump gRPC & GCSIO --- end2end-test-examples/gcs/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 4a07dd4b..2618a64d 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -9,8 +9,8 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 -def gcsioVersion = '2.2.5' -def grpcVersion = '1.44.0' +def gcsioVersion = '2.2.6' +def grpcVersion = '1.46.0' def protobufVersion = '3.19.2' def protocVersion = protobufVersion def conscryptVersion = '2.5.2' From 78925bfb305686d74368041cd0b15de505c9cb59 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 17:09:39 -0700 Subject: [PATCH 51/87] Added ObjectResolver --- .../gcs/src/main/java/io/grpc/gcs/Args.java | 23 +++-- .../src/main/java/io/grpc/gcs/GrpcClient.java | 96 +++++++++---------- .../main/java/io/grpc/gcs/ObjectResolver.java | 24 +++++ 3 files changed, 83 insertions(+), 60 deletions(-) create mode 100644 end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ObjectResolver.java diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java index af0bd4af..46660991 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/Args.java @@ -26,7 +26,11 @@ public class Args { final int port; final String service_path; final String access_token; - final String bkt, obj; + final String bkt; + final String obj; + final String objFormat; + final int objStart; + final int objStop; final boolean dp; final boolean rr; final boolean td; @@ -45,11 +49,8 @@ public class Args { final int zeroCopy; // 0=auto, 1=on, -1=off Args(String[] args) throws ArgumentParserException { - ArgumentParser parser = - ArgumentParsers.newFor("GCS client test") - .build() - .defaultHelp(true) - .description("GCS client java binary"); + ArgumentParser parser = ArgumentParsers.newFor("GCS client test").build().defaultHelp(true) + .description("GCS client java binary"); parser.addArgument("--calls").type(Integer.class).setDefault(1); parser.addArgument("--warmups").type(Integer.class).setDefault(0); @@ -59,8 +60,11 @@ public class Args { parser.addArgument("--port").type(Integer.class).setDefault(PORT); parser.addArgument("--service_path").type(String.class).setDefault("storage/v1/"); parser.addArgument("--access_token").type(String.class).setDefault(""); - parser.addArgument("--bkt").type(String.class).setDefault("gcs-grpc-team-weiranf"); - parser.addArgument("--obj").type(String.class).setDefault("a"); + parser.addArgument("--bkt").type(String.class).setDefault("gcs-grpc-team-veblush1"); + parser.addArgument("--obj").type(String.class).setDefault("1G"); + parser.addArgument("--obj_format").type(String.class).setDefault(""); + parser.addArgument("--obj_start").type(Integer.class).setDefault(0); + parser.addArgument("--obj_stop").type(Integer.class).setDefault(0); parser.addArgument("--dp").type(Boolean.class).setDefault(false); parser.addArgument("--rr").type(Boolean.class).setDefault(false); parser.addArgument("--td").type(Boolean.class).setDefault(false); @@ -91,6 +95,9 @@ public class Args { access_token = ns.getString("access_token"); bkt = ns.getString("bkt"); obj = ns.getString("obj"); + objFormat = ns.getString("obj_format"); + objStart = ns.getInt("obj_start"); + objStop = ns.getInt("obj_stop"); dp = ns.getBoolean("dp"); rr = ns.getBoolean("rr"); td = ns.getBoolean("td"); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 3fe0d86d..bee6f5fa 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -48,10 +48,10 @@ public class GrpcClient { new ZeroCopyMessageMarshaller(ReadObjectResponse.getDefaultInstance()); private static final MethodDescriptor readObjectMethod = StorageGrpc.getReadObjectMethod().toBuilder() - .setResponseMarshaller(ReadObjectResponseMarshaller) - .build(); + .setResponseMarshaller(ReadObjectResponseMarshaller).build(); private final boolean useZeroCopy; + private ObjectResolver objectResolver; private ManagedChannel[] channels; private Args args; private GoogleCredentials creds; @@ -65,6 +65,7 @@ private static String toV2BucketName(String v1BucketName) { public GrpcClient(Args args) throws IOException { this.args = args; + this.objectResolver = new ObjectResolver(args.obj, args.objFormat, args.objStart, args.objStop); if (args.access_token.equals("")) { this.creds = GoogleCredentials.getApplicationDefault().createScoped(SCOPE); } else if (args.access_token.equals("-")) { @@ -87,13 +88,13 @@ public GrpcClient(Args args) throws IOException { ImmutableMap policyStrategy = ImmutableMap.of(policy, ImmutableMap.of()); ImmutableMap childPolicy = - ImmutableMap.of( - "childPolicy", ImmutableList.of(policyStrategy)); + ImmutableMap.of("childPolicy", + ImmutableList.of(policyStrategy)); ImmutableMap grpcLbPolicy = ImmutableMap.of("grpclb", childPolicy); ImmutableMap loadBalancingConfig = - ImmutableMap.of( - "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); + ImmutableMap.of("loadBalancingConfig", + ImmutableList.of(grpcLbPolicy)); gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); if (args.flowControlWindow > 0) { @@ -112,7 +113,8 @@ public GrpcClient(Args args) throws IOException { } channelBuilder = gceChannelBuilder; } else { - NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(args.host, args.port); + NettyChannelBuilder nettyChannelBuilder = + NettyChannelBuilder.forAddress(args.host, args.port); if (args.flowControlWindow > 0) { nettyChannelBuilder.flowControlWindow(args.flowControlWindow); } @@ -148,13 +150,13 @@ public void startCalls(ResultTable results) throws InterruptedException { try { switch (args.method) { case METHOD_READ: - makeReadObjectRequest(channel, results); + makeReadObjectRequest(channel, results, 1); break; case METHOD_RANDOM: - makeRandomReadRequest(channel, results); + makeRandomReadRequest(channel, results, 1); break; case METHOD_WRITE: - makeInsertRequest(channel, results, 0); + makeInsertRequest(channel, results, 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -170,28 +172,29 @@ public void startCalls(ResultTable results) throws InterruptedException { case METHOD_READ: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> makeReadObjectRequest(this.channels[finalI], results); + Runnable task = + () -> makeReadObjectRequest(this.channels[finalI], results, finalI + 1); threadPoolExecutor.execute(task); } break; case METHOD_RANDOM: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> makeRandomReadRequest(this.channels[finalI], results); + Runnable task = + () -> makeRandomReadRequest(this.channels[finalI], results, finalI + 1); threadPoolExecutor.execute(task); } break; case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = - () -> { - try { - makeInsertRequest(this.channels[finalI], results, finalI); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = () -> { + try { + makeInsertRequest(this.channels[finalI], results, finalI + 1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -207,28 +210,24 @@ public void startCalls(ResultTable results) throws InterruptedException { } } - private void makeReadObjectRequest(ManagedChannel channel, ResultTable results) { + private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, int threadId) { StorageGrpc.StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } - ReadObjectRequest readRequest = - ReadObjectRequest.newBuilder() - .setBucket(toV2BucketName(args.bkt)) - .setObject(args.obj) - .build(); byte[] scratch = new byte[4 * 1024 * 1024]; for (int i = 0; i < args.calls; i++) { + ReadObjectRequest readRequest = + ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)) + .setObject(objectResolver.Resolve(threadId, i)).build(); + long start = System.currentTimeMillis(); Iterator resIterator; if (useZeroCopy) { resIterator = - io.grpc.stub.ClientCalls.blockingServerStreamingCall( - blockingStub.getChannel(), - readObjectMethod, - blockingStub.getCallOptions(), - readRequest); + io.grpc.stub.ClientCalls.blockingServerStreamingCall(blockingStub.getChannel(), + readObjectMethod, blockingStub.getCallOptions(), readRequest); } else { resIterator = blockingStub.readObject(readRequest); } @@ -260,14 +259,14 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results) } } - private void makeRandomReadRequest(ManagedChannel channel, ResultTable results) { + private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, int threadId) { StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel); if (creds != null) { blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } - ReadObjectRequest.Builder reqBuilder = - ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(args.obj); + ReadObjectRequest.Builder reqBuilder = ReadObjectRequest.newBuilder() + .setBucket(toV2BucketName(args.bkt)).setObject(objectResolver.Resolve(threadId, 0)); Random r = new Random(); long buffSize = args.buffSize * 1024; @@ -291,18 +290,13 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results) long dur = System.currentTimeMillis() - start; logger.info("time cost for getObjectMedia: " + dur + "ms"); logger.info("total iterations: " + itr); - logger.info( - "start pos: " - + offset - + ", read lenth: " - + buffSize - + ", total KB read: " - + bytesRead / 1024); + logger.info("start pos: " + offset + ", read lenth: " + buffSize + ", total KB read: " + + bytesRead / 1024); results.reportResult(dur); } } - private void makeInsertRequest(ManagedChannel channel, ResultTable results, int idx) + private void makeInsertRequest(ManagedChannel channel, ResultTable results, int threadId) throws InterruptedException { StorageGrpc.StorageStub asyncStub = StorageGrpc.newStub(channel); if (creds != null) { @@ -312,6 +306,8 @@ private void makeInsertRequest(ManagedChannel channel, ResultTable results, int int totalBytes = args.size * 1024; byte[] data = new byte[totalBytes]; for (int i = 0; i < args.calls; i++) { + String obj = objectResolver.Resolve(threadId, i); + int offset = 0; boolean isFirst = true; boolean isLast = false; @@ -360,7 +356,7 @@ public void onCompleted() { } WriteObjectRequest req = - getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), idx); + getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), obj); requestObserver.onNext(req); if (finishLatch.getCount() == 0) { logger.warning("Stream completed before finishing sending requests"); @@ -377,17 +373,13 @@ public void onCompleted() { } } - private WriteObjectRequest getWriteRequest( - boolean first, boolean last, int offset, ByteString bytes, int idx) { + private WriteObjectRequest getWriteRequest(boolean first, boolean last, int offset, + ByteString bytes, String obj) { WriteObjectRequest.Builder builder = WriteObjectRequest.newBuilder(); if (first) { - builder.setWriteObjectSpec( - WriteObjectSpec.newBuilder() - .setResource( - Object.newBuilder() - .setBucket(toV2BucketName(args.bkt)) - .setName(args.obj + "_" + idx)) - .build()); + builder.setWriteObjectSpec(WriteObjectSpec.newBuilder() + .setResource(Object.newBuilder().setBucket(toV2BucketName(args.bkt)).setName(obj)) + .build()); } builder.setChecksummedData(ChecksummedData.newBuilder().setContent(bytes).build()); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ObjectResolver.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ObjectResolver.java new file mode 100644 index 00000000..f7cdba26 --- /dev/null +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ObjectResolver.java @@ -0,0 +1,24 @@ +package io.grpc.gcs; + +public class ObjectResolver { + public ObjectResolver(String object, String objectFormat, int objectStart, int objectStop) { + this.object = object; + this.objectFormat = objectFormat; + this.objectStart = objectStart; + this.objectStop = objectStop; + } + + public String Resolve(int threadId, int objectId) { + if (objectFormat == null || objectFormat.equals("")) { + return object; + } + int oid = objectStop == 0 ? objectId : (objectId % (objectStop - objectStart)) + objectStart; + return objectFormat.replaceAll("\\{t\\}", String.valueOf(threadId)).replaceAll("\\{o\\}", + String.valueOf(oid)); + } + + private String object; + private String objectFormat; + private int objectStart; + private int objectStop; +} From 2be14d1fed0dc5b0ae8d52f8ce9852bef431fbc2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 17:26:26 -0700 Subject: [PATCH 52/87] Apply object-resolver to all GCS clients --- .../main/java/io/grpc/gcs/GcsioClient.java | 157 +++++++++--------- .../src/main/java/io/grpc/gcs/GrpcClient.java | 10 +- .../src/main/java/io/grpc/gcs/HttpClient.java | 43 +++-- 3 files changed, 110 insertions(+), 100 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index a9d6720c..5aeed039 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -29,11 +29,13 @@ public class GcsioClient { private static final String SCOPE = "https://www.googleapis.com/auth/cloud-platform"; private Args args; + private ObjectResolver objectResolver; private GoogleCloudStorageOptions gcsOpts; private GoogleCredential creds; public GcsioClient(Args args, boolean grpcEnabled) throws IOException { this.args = args; + this.objectResolver = new ObjectResolver(args.obj, args.objFormat, args.objStart, args.objStop); if (args.access_token.equals("")) { this.creds = GoogleCredential.getApplicationDefault().createScoped(Arrays.asList(SCOPE)); } else if (args.access_token.equals("-")) { @@ -42,20 +44,14 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { logger.warning("Please provide valid --access_token"); } - GoogleCloudStorageOptions.Builder optsBuilder = - GoogleCloudStorageOptions.builder() - .setAppName("weiranf-app") - .setGrpcEnabled(grpcEnabled) - .setStorageRootUrl("https://" + args.host) - .setStorageServicePath(args.service_path) - .setTrafficDirectorEnabled(args.td) - .setDirectPathPreferred(args.dp) - .setReadChannelOptions( - GoogleCloudStorageReadOptions.builder() - .setGrpcChecksumsEnabled(args.checksum) - .build()) - .setWriteChannelOptions( - AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()); + GoogleCloudStorageOptions.Builder optsBuilder = GoogleCloudStorageOptions.builder() + .setAppName("weiranf-app").setGrpcEnabled(grpcEnabled) + .setStorageRootUrl("https://" + args.host).setStorageServicePath(args.service_path) + .setTrafficDirectorEnabled(args.td).setDirectPathPreferred(args.dp) + .setReadChannelOptions( + GoogleCloudStorageReadOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) + .setWriteChannelOptions( + AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()); if (!Strings.isNullOrEmpty(args.host2)) { optsBuilder.setGrpcServerAddress(args.host2); } @@ -66,13 +62,13 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep if (args.threads == 1) { switch (args.method) { case METHOD_READ: - makeMediaRequest(results); - break; - case METHOD_WRITE: - makeWriteRequest(results, 0); + makeMediaRequest(results, 1); break; case METHOD_RANDOM: - makeRandomMediaRequest(results); + makeRandomMediaRequest(results, 1); + break; + case METHOD_WRITE: + makeWriteRequest(results, 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -84,28 +80,40 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep switch (args.method) { case METHOD_READ: for (int i = 0; i < args.threads; i++) { - Runnable task = - () -> { - try { - makeMediaRequest(results); - } catch (IOException e) { - e.printStackTrace(); - } - }; + int finalI = i; + Runnable task = () -> { + try { + makeMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; + threadPoolExecutor.execute(task); + } + break; + case METHOD_RANDOM: + for (int i = 0; i < args.threads; i++) { + int finalI = i; + Runnable task = () -> { + try { + makeRandomMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = - () -> { - try { - makeWriteRequest(results, finalI); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = () -> { + try { + makeWriteRequest(results, finalI + 1); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -121,19 +129,15 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep } } - private void makeMediaRequest(ResultTable results) throws IOException { - GoogleCloudStorageFileSystem gcsfs = - new GoogleCloudStorageFileSystem( - creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + private void makeMediaRequest(ResultTable results, int threadId) throws IOException { + GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); int size = args.size * 1024; - - URI uri = URI.create("gs://" + args.bkt + "/" + args.obj); - ByteBuffer buff = ByteBuffer.allocate(size); for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); + URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, i)); ReadableByteChannel readChannel = gcsfs.open(uri); readChannel.read(buff); long dur = System.currentTimeMillis() - start; @@ -142,53 +146,19 @@ private void makeMediaRequest(ResultTable results) throws IOException { } buff.clear(); readChannel.close(); - // logger.info("time cost for reading bytes: " + dur + "ms"); - results.reportResult(dur); - } - - gcsfs.close(); - } - - private void makeWriteRequest(ResultTable results, int idx) - throws IOException, InterruptedException { - GoogleCloudStorageFileSystem gcsfs = - new GoogleCloudStorageFileSystem( - creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); - - int size = args.size * 1024; - Random rd = new Random(); - byte[] randBytes = new byte[size]; - rd.nextBytes(randBytes); - - URI uri = URI.create("gs://" + args.bkt + "/" + args.obj + "_" + idx); - for (int i = 0; i < args.calls; i++) { - long start = System.currentTimeMillis(); - WritableByteChannel writeChannel = gcsfs.create(uri); - ByteBuffer buff = ByteBuffer.wrap(randBytes); - writeChannel.write(buff); - writeChannel.close(); - // write operation is async, need to call close() to wait for finish. - long dur = System.currentTimeMillis() - start; results.reportResult(dur); - if (dur < 1000) { - Thread.sleep(1000 - dur); // Avoid limit of 1 qps for updating the same object - } } gcsfs.close(); } - private void makeRandomMediaRequest(ResultTable results) throws IOException { - GoogleCloudStorageFileSystem gcsfs = - new GoogleCloudStorageFileSystem( - creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + private void makeRandomMediaRequest(ResultTable results, int threadId) throws IOException { + GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); Random r = new Random(); - URI uri = URI.create("gs://" + args.bkt + "/" + args.obj); - + URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, 0)); GoogleCloudStorageReadOptions readOpts = gcsOpts.getReadChannelOptions(); SeekableByteChannel reader = gcsfs.open(uri, readOpts); @@ -209,4 +179,29 @@ private void makeRandomMediaRequest(ResultTable results) throws IOException { gcsfs.close(); } + + private void makeWriteRequest(ResultTable results, int threadId) + throws IOException, InterruptedException { + GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + + int size = args.size * 1024; + Random rd = new Random(); + byte[] randBytes = new byte[size]; + rd.nextBytes(randBytes); + + for (int i = 0; i < args.calls; i++) { + long start = System.currentTimeMillis(); + URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, i)); + WritableByteChannel writeChannel = gcsfs.create(uri); + ByteBuffer buff = ByteBuffer.wrap(randBytes); + writeChannel.write(buff); + writeChannel.close(); + // write operation is async, need to call close() to wait for finish. + long dur = System.currentTimeMillis() - start; + results.reportResult(dur); + } + + gcsfs.close(); + } } diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index bee6f5fa..93958034 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -51,9 +51,9 @@ public class GrpcClient { .setResponseMarshaller(ReadObjectResponseMarshaller).build(); private final boolean useZeroCopy; + private Args args; private ObjectResolver objectResolver; private ManagedChannel[] channels; - private Args args; private GoogleCredentials creds; private static final String SCOPE = "https://www.googleapis.com/auth/cloud-platform"; @@ -330,14 +330,6 @@ public void onError(Throwable t) { public void onCompleted() { long dur = System.currentTimeMillis() - start; results.reportResult(dur); - if (dur < 1000) { - try { - Thread.sleep(1000 - dur); // Avoid limit of 1 qps for updating the same object - } catch (InterruptedException e) { - e.printStackTrace(); - finishLatch.countDown(); - } - } finishLatch.countDown(); } }; diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java index c407668d..30d45c47 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java @@ -21,10 +21,12 @@ public class HttpClient { private static final Logger logger = Logger.getLogger(HttpClient.class.getName()); private Args args; + private ObjectResolver objectResolver; private Storage client; public HttpClient(Args args) { this.args = args; + this.objectResolver = new ObjectResolver(args.obj, args.objFormat, args.objStart, args.objStop); this.client = StorageOptions.getDefaultInstance().getService(); } @@ -32,13 +34,13 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep if (args.threads == 0) { switch (args.method) { case METHOD_READ: - makeMediaRequest(results); + makeMediaRequest(results, 1); break; case METHOD_RANDOM: - makeRandomMediaRequest(results); + makeRandomMediaRequest(results, 1); break; case METHOD_WRITE: - makeInsertRequest(results); + makeInsertRequest(results, 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -49,7 +51,28 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep switch (args.method) { case METHOD_READ: for (int i = 0; i < args.threads; i++) { - Runnable task = () -> makeMediaRequest(results); + int finalI = i; + Runnable task = () -> makeMediaRequest(results, finalI + 1); + threadPoolExecutor.execute(task); + } + break; + case METHOD_RANDOM: + for (int i = 0; i < args.threads; i++) { + int finalI = i; + Runnable task = () -> { + try { + makeRandomMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; + threadPoolExecutor.execute(task); + } + break; + case METHOD_WRITE: + for (int i = 0; i < args.threads; i++) { + int finalI = i; + Runnable task = () -> makeInsertRequest(results, finalI + 1); threadPoolExecutor.execute(task); } break; @@ -63,9 +86,9 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep } } - public void makeMediaRequest(ResultTable results) { - BlobId blobId = BlobId.of(args.bkt, args.obj); + public void makeMediaRequest(ResultTable results, int threadId) { for (int i = 0; i < args.calls; i++) { + BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, i)); long start = System.currentTimeMillis(); byte[] content = client.readAllBytes(blobId); // String contentString = new String(content, UTF_8); @@ -77,10 +100,10 @@ public void makeMediaRequest(ResultTable results) { } } - public void makeRandomMediaRequest(ResultTable results) throws IOException { + public void makeRandomMediaRequest(ResultTable results, int threadId) throws IOException { Random r = new Random(); - BlobId blobId = BlobId.of(args.bkt, args.obj); + BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, 0)); ReadChannel reader = client.reader(blobId); for (int i = 0; i < args.calls; i++) { long offset = (long) r.nextInt(args.size - args.buffSize) * 1024; @@ -101,11 +124,11 @@ public void makeRandomMediaRequest(ResultTable results) throws IOException { reader.close(); } - public void makeInsertRequest(ResultTable results) { + public void makeInsertRequest(ResultTable results, int threadId) { int totalBytes = args.size * 1024; byte[] data = new byte[totalBytes]; - BlobId blobId = BlobId.of(args.bkt, args.obj); for (int i = 0; i < args.calls; i++) { + BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, i)); long start = System.currentTimeMillis(); client.create(BlobInfo.newBuilder(blobId).build(), data); long dur = System.currentTimeMillis() - start; From 82a682738a88783d3e3a52d6cae20dc31bbff3f7 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 17:37:59 -0700 Subject: [PATCH 53/87] Clean up log --- .../gcs/src/main/java/io/grpc/gcs/GrpcClient.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 93958034..dc4269ba 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -270,7 +270,7 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, Random r = new Random(); long buffSize = args.buffSize * 1024; - + byte[] scratch = new byte[4 * 1024 * 1024]; for (int i = 0; i < args.calls; i++) { long offset = (long) r.nextInt(args.size - args.buffSize) * 1024; reqBuilder.setReadOffset(offset); @@ -279,19 +279,12 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, long start = System.currentTimeMillis(); Iterator resIterator = blockingStub.readObject(req); - int itr = 0; - long bytesRead = 0; while (resIterator.hasNext()) { - itr++; ReadObjectResponse res = resIterator.next(); - bytesRead += res.getChecksummedData().getSerializedSize(); - // logger.info("result: " + res.getChecksummedData()); + ByteString content = res.getChecksummedData().getContent(); + content.copyTo(scratch, 0); } long dur = System.currentTimeMillis() - start; - logger.info("time cost for getObjectMedia: " + dur + "ms"); - logger.info("total iterations: " + itr); - logger.info("start pos: " + offset + ", read lenth: " + buffSize + ", total KB read: " - + bytesRead / 1024); results.reportResult(dur); } } From f3e8a0bb38e2a160500724b718238074bb99eb9a Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 23:17:19 -0700 Subject: [PATCH 54/87] More result --- .../main/java/io/grpc/gcs/GcsioClient.java | 16 +++--- .../src/main/java/io/grpc/gcs/GrpcClient.java | 19 ++++--- .../src/main/java/io/grpc/gcs/HttpClient.java | 15 +++-- .../main/java/io/grpc/gcs/ResultTable.java | 57 +++++++------------ 4 files changed, 50 insertions(+), 57 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 5aeed039..4185ed0f 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -137,7 +137,8 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept ByteBuffer buff = ByteBuffer.allocate(size); for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); - URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, i)); + String object = objectResolver.Resolve(threadId, i); + URI uri = URI.create("gs://" + args.bkt + "/" + object); ReadableByteChannel readChannel = gcsfs.open(uri); readChannel.read(buff); long dur = System.currentTimeMillis() - start; @@ -146,7 +147,7 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept } buff.clear(); readChannel.close(); - results.reportResult(dur); + results.reportResult(args.bkt, object, size, dur); } gcsfs.close(); @@ -158,7 +159,8 @@ private void makeRandomMediaRequest(ResultTable results, int threadId) throws IO Random r = new Random(); - URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, 0)); + String object = objectResolver.Resolve(threadId, 0); + URI uri = URI.create("gs://" + args.bkt + "/" + object); GoogleCloudStorageReadOptions readOpts = gcsOpts.getReadChannelOptions(); SeekableByteChannel reader = gcsfs.open(uri, readOpts); @@ -172,8 +174,7 @@ private void makeRandomMediaRequest(ResultTable results, int threadId) throws IO if (buff.remaining() > 0) { logger.warning("Got remaining bytes: " + buff.remaining()); } - logger.info("time cost for reading bytes: " + dur + "ms"); - results.reportResult(dur); + results.reportResult(args.bkt, object, args.buffSize * 1024, dur); } reader.close(); @@ -192,14 +193,15 @@ private void makeWriteRequest(ResultTable results, int threadId) for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); - URI uri = URI.create("gs://" + args.bkt + "/" + objectResolver.Resolve(threadId, i)); + String object = objectResolver.Resolve(threadId, 0); + URI uri = URI.create("gs://" + args.bkt + "/" + object); WritableByteChannel writeChannel = gcsfs.create(uri); ByteBuffer buff = ByteBuffer.wrap(randBytes); writeChannel.write(buff); writeChannel.close(); // write operation is async, need to call close() to wait for finish. long dur = System.currentTimeMillis() - start; - results.reportResult(dur); + results.reportResult(args.bkt, object, size, dur); } gcsfs.close(); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index dc4269ba..2342c4e1 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -218,11 +218,12 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, byte[] scratch = new byte[4 * 1024 * 1024]; for (int i = 0; i < args.calls; i++) { - ReadObjectRequest readRequest = - ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)) - .setObject(objectResolver.Resolve(threadId, i)).build(); + String object = objectResolver.Resolve(threadId, i); + ReadObjectRequest readRequest = ReadObjectRequest.newBuilder() + .setBucket(toV2BucketName(args.bkt)).setObject(object).build(); long start = System.currentTimeMillis(); + long totalBytes = 0; Iterator resIterator; if (useZeroCopy) { resIterator = @@ -241,6 +242,7 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, try { // Just copy to scratch memory to ensure its data is consumed. ByteString content = res.getChecksummedData().getContent(); + totalBytes += content.size(); content.copyTo(scratch, 0); } finally { if (stream != null) { @@ -255,7 +257,7 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, } catch (NoSuchElementException e) { } long dur = System.currentTimeMillis() - start; - results.reportResult(dur); + results.reportResult(args.bkt, object, totalBytes, dur); } } @@ -265,8 +267,9 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } - ReadObjectRequest.Builder reqBuilder = ReadObjectRequest.newBuilder() - .setBucket(toV2BucketName(args.bkt)).setObject(objectResolver.Resolve(threadId, 0)); + String object = objectResolver.Resolve(threadId, 0); + ReadObjectRequest.Builder reqBuilder = + ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(object); Random r = new Random(); long buffSize = args.buffSize * 1024; @@ -285,7 +288,7 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, content.copyTo(scratch, 0); } long dur = System.currentTimeMillis() - start; - results.reportResult(dur); + results.reportResult(args.bkt, object, buffSize, dur); } } @@ -322,7 +325,7 @@ public void onError(Throwable t) { @Override public void onCompleted() { long dur = System.currentTimeMillis() - start; - results.reportResult(dur); + results.reportResult(args.bkt, obj, totalBytes, dur); finishLatch.countDown(); } }; diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java index 30d45c47..8f4f06b3 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java @@ -88,7 +88,8 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep public void makeMediaRequest(ResultTable results, int threadId) { for (int i = 0; i < args.calls; i++) { - BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, i)); + String object = objectResolver.Resolve(threadId, i); + BlobId blobId = BlobId.of(args.bkt, object); long start = System.currentTimeMillis(); byte[] content = client.readAllBytes(blobId); // String contentString = new String(content, UTF_8); @@ -96,14 +97,15 @@ public void makeMediaRequest(ResultTable results, int threadId) { long dur = System.currentTimeMillis() - start; // logger.info("time cost for readAllBytes: " + dur + "ms"); // logger.info("total KB received: " + content.length/1024); - results.reportResult(dur); + results.reportResult(args.bkt, object, content.length, dur); } } public void makeRandomMediaRequest(ResultTable results, int threadId) throws IOException { Random r = new Random(); - BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, 0)); + String object = objectResolver.Resolve(threadId, 0); + BlobId blobId = BlobId.of(args.bkt, object); ReadChannel reader = client.reader(blobId); for (int i = 0; i < args.calls; i++) { long offset = (long) r.nextInt(args.size - args.buffSize) * 1024; @@ -119,7 +121,7 @@ public void makeRandomMediaRequest(ResultTable results, int threadId) throws IOE logger.info("total KB received: " + buff.position() / 1024); logger.info("time cost for random reading: " + dur + "ms"); buff.clear(); - results.reportResult(dur); + results.reportResult(args.bkt, object, args.buffSize * 1024, dur); } reader.close(); } @@ -128,12 +130,13 @@ public void makeInsertRequest(ResultTable results, int threadId) { int totalBytes = args.size * 1024; byte[] data = new byte[totalBytes]; for (int i = 0; i < args.calls; i++) { - BlobId blobId = BlobId.of(args.bkt, objectResolver.Resolve(threadId, i)); + String object = objectResolver.Resolve(threadId, i); + BlobId blobId = BlobId.of(args.bkt, object); long start = System.currentTimeMillis(); client.create(BlobInfo.newBuilder(blobId).build(), data); long dur = System.currentTimeMillis() - start; logger.info("time cost for creating blob: " + dur + "ms"); - results.reportResult(dur); + results.reportResult(args.bkt, object, totalBytes, dur); } } } diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java index 3a133f88..b830bff3 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/ResultTable.java @@ -39,7 +39,7 @@ public void stop() { } } - public void reportResult(long duration) { + public void reportResult(String bucket, String object, long bytes, long duration) { int ord; synchronized (this) { Result result = new Result(); @@ -49,9 +49,8 @@ public void reportResult(long duration) { ord = results.size(); } if (this.args.verboseResult) { - System.out.format( - "### Result: ord=%d elapsed=%d%s\n", - ord, duration, results.size() <= warmupCount ? " [WARM-UP]" : ""); + System.out.format("### Result: ord=%d bucket=%s object=%s bytes=%d elapsed=%d%s\n", ord, + bucket, object, bytes, duration, results.size() <= warmupCount ? " [WARM-UP]" : ""); System.out.flush(); } } @@ -65,21 +64,20 @@ public void printResult() throws IOException { synchronized (this) { results.subList(0, Math.min(results.size(), warmupCount)).clear(); - if (results.size() == 0) return; + if (results.size() == 0) + return; int n = results.size(); long totalDur = results.get(n - 1).startTime + results.get(n - 1).duration - results.get(0).startTime; double totalSec = totalDur / 1000.0; - Collections.sort( - results, - new Comparator() { - @Override - public int compare(Result o1, Result o2) { - return Long.compare(o1.duration, o2.duration); - } - }); + Collections.sort(results, new Comparator() { + @Override + public int compare(Result o1, Result o2) { + return Long.compare(o1.duration, o2.duration); + } + }); Gson gson = new Gson(); BenchmarkResult benchmarkResult = new BenchmarkResult(); @@ -94,29 +92,16 @@ public int compare(Result o1, Result o2) { gson.toJson(benchmarkResult, writer); writer.close(); } - System.out.println( - String.format( - "****** Test Results [client: %s, method: %s, size: %d, threads: %d, dp: %s, calls:" - + " %d, qps: %f]: \n" - + "\t\tMin\tp5\tp10\tp25\tp50\tp75\tp90\tp99\tMax\tTotal\n" - + " Time(ms)\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", - args.client, - args.method, - args.size, - args.threads, - args.dp, - n, - n / totalSec, - results.get(0).duration, - results.get((int) (n * 0.05)).duration, - results.get((int) (n * 0.1)).duration, - results.get((int) (n * 0.25)).duration, - results.get((int) (n * 0.50)).duration, - results.get((int) (n * 0.75)).duration, - results.get((int) (n * 0.90)).duration, - results.get((int) (n * 0.99)).duration, - results.get(n - 1).duration, - totalDur)); + System.out.println(String.format( + "****** Test Results [client: %s, method: %s, size: %d, threads: %d, dp: %s, calls:" + + " %d, qps: %f]: \n" + "\t\tMin\tp5\tp10\tp25\tp50\tp75\tp90\tp99\tMax\tTotal\n" + + " Time(ms)\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", + args.client, args.method, args.size, args.threads, args.dp, n, n / totalSec, + results.get(0).duration, results.get((int) (n * 0.05)).duration, + results.get((int) (n * 0.1)).duration, results.get((int) (n * 0.25)).duration, + results.get((int) (n * 0.50)).duration, results.get((int) (n * 0.75)).duration, + results.get((int) (n * 0.90)).duration, results.get((int) (n * 0.99)).duration, + results.get(n - 1).duration, totalDur)); } } } From ffc955a1c43992691f7693fd0aaba59d2ddfb5b0 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 7 May 2022 23:21:08 -0700 Subject: [PATCH 55/87] Fix index --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 4185ed0f..2c2dc767 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -193,7 +193,7 @@ private void makeWriteRequest(ResultTable results, int threadId) for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); - String object = objectResolver.Resolve(threadId, 0); + String object = objectResolver.Resolve(threadId, i); URI uri = URI.create("gs://" + args.bkt + "/" + object); WritableByteChannel writeChannel = gcsfs.create(uri); ByteBuffer buff = ByteBuffer.wrap(randBytes); From b47920bb70f4412611e1da1820c30fd07c9e070a Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 10 May 2022 15:47:45 -0700 Subject: [PATCH 56/87] Update by review --- .../main/java/io/grpc/gcs/GcsioClient.java | 95 +++++++++++-------- .../src/main/java/io/grpc/gcs/GrpcClient.java | 59 +++++++----- .../src/main/java/io/grpc/gcs/HttpClient.java | 23 ++--- 3 files changed, 101 insertions(+), 76 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 2c2dc767..8f87cfa5 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -44,14 +44,20 @@ public GcsioClient(Args args, boolean grpcEnabled) throws IOException { logger.warning("Please provide valid --access_token"); } - GoogleCloudStorageOptions.Builder optsBuilder = GoogleCloudStorageOptions.builder() - .setAppName("weiranf-app").setGrpcEnabled(grpcEnabled) - .setStorageRootUrl("https://" + args.host).setStorageServicePath(args.service_path) - .setTrafficDirectorEnabled(args.td).setDirectPathPreferred(args.dp) - .setReadChannelOptions( - GoogleCloudStorageReadOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()) - .setWriteChannelOptions( - AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()); + GoogleCloudStorageOptions.Builder optsBuilder = + GoogleCloudStorageOptions.builder() + .setAppName("weiranf-app") + .setGrpcEnabled(grpcEnabled) + .setStorageRootUrl("https://" + args.host) + .setStorageServicePath(args.service_path) + .setTrafficDirectorEnabled(args.td) + .setDirectPathPreferred(args.dp) + .setReadChannelOptions( + GoogleCloudStorageReadOptions.builder() + .setGrpcChecksumsEnabled(args.checksum) + .build()) + .setWriteChannelOptions( + AsyncWriteChannelOptions.builder().setGrpcChecksumsEnabled(args.checksum).build()); if (!Strings.isNullOrEmpty(args.host2)) { optsBuilder.setGrpcServerAddress(args.host2); } @@ -62,13 +68,13 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep if (args.threads == 1) { switch (args.method) { case METHOD_READ: - makeMediaRequest(results, 1); + makeMediaRequest(results, /*threadId=*/ 1); break; case METHOD_RANDOM: - makeRandomMediaRequest(results, 1); + makeRandomMediaRequest(results, /*threadId=*/ 1); break; case METHOD_WRITE: - makeWriteRequest(results, 1); + makeWriteRequest(results, /*threadId=*/ 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -81,39 +87,42 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep case METHOD_READ: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeMediaRequest(results, finalI + 1); - } catch (IOException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; case METHOD_RANDOM: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeRandomMediaRequest(results, finalI + 1); - } catch (IOException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeRandomMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeWriteRequest(results, finalI + 1); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeWriteRequest(results, finalI + 1); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -130,8 +139,10 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep } private void makeMediaRequest(ResultTable results, int threadId) throws IOException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); int size = args.size * 1024; ByteBuffer buff = ByteBuffer.allocate(size); @@ -154,12 +165,14 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept } private void makeRandomMediaRequest(ResultTable results, int threadId) throws IOException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); Random r = new Random(); - String object = objectResolver.Resolve(threadId, 0); + String object = objectResolver.Resolve(threadId, /*objectId=*/ 0); URI uri = URI.create("gs://" + args.bkt + "/" + object); GoogleCloudStorageReadOptions readOpts = gcsOpts.getReadChannelOptions(); @@ -183,8 +196,10 @@ private void makeRandomMediaRequest(ResultTable results, int threadId) throws IO private void makeWriteRequest(ResultTable results, int threadId) throws IOException, InterruptedException { - GoogleCloudStorageFileSystem gcsfs = new GoogleCloudStorageFileSystem(creds, - GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); + GoogleCloudStorageFileSystem gcsfs = + new GoogleCloudStorageFileSystem( + creds, + GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); int size = args.size * 1024; Random rd = new Random(); @@ -198,8 +213,8 @@ private void makeWriteRequest(ResultTable results, int threadId) WritableByteChannel writeChannel = gcsfs.create(uri); ByteBuffer buff = ByteBuffer.wrap(randBytes); writeChannel.write(buff); - writeChannel.close(); // write operation is async, need to call close() to wait for finish. + writeChannel.close(); long dur = System.currentTimeMillis() - start; results.reportResult(args.bkt, object, size, dur); } diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java index 2342c4e1..74323894 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GrpcClient.java @@ -48,7 +48,8 @@ public class GrpcClient { new ZeroCopyMessageMarshaller(ReadObjectResponse.getDefaultInstance()); private static final MethodDescriptor readObjectMethod = StorageGrpc.getReadObjectMethod().toBuilder() - .setResponseMarshaller(ReadObjectResponseMarshaller).build(); + .setResponseMarshaller(ReadObjectResponseMarshaller) + .build(); private final boolean useZeroCopy; private Args args; @@ -88,13 +89,13 @@ public GrpcClient(Args args) throws IOException { ImmutableMap policyStrategy = ImmutableMap.of(policy, ImmutableMap.of()); ImmutableMap childPolicy = - ImmutableMap.of("childPolicy", - ImmutableList.of(policyStrategy)); + ImmutableMap.of( + "childPolicy", ImmutableList.of(policyStrategy)); ImmutableMap grpcLbPolicy = ImmutableMap.of("grpclb", childPolicy); ImmutableMap loadBalancingConfig = - ImmutableMap.of("loadBalancingConfig", - ImmutableList.of(grpcLbPolicy)); + ImmutableMap.of( + "loadBalancingConfig", ImmutableList.of(grpcLbPolicy)); gceChannelBuilder.defaultServiceConfig(loadBalancingConfig); if (args.flowControlWindow > 0) { @@ -150,13 +151,13 @@ public void startCalls(ResultTable results) throws InterruptedException { try { switch (args.method) { case METHOD_READ: - makeReadObjectRequest(channel, results, 1); + makeReadObjectRequest(channel, results, /*threadId=*/ 1); break; case METHOD_RANDOM: - makeRandomReadRequest(channel, results, 1); + makeRandomReadRequest(channel, results, /*threadId=*/ 1); break; case METHOD_WRITE: - makeInsertRequest(channel, results, 1); + makeInsertRequest(channel, results, /*threadId=*/ 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -188,13 +189,14 @@ public void startCalls(ResultTable results) throws InterruptedException { case METHOD_WRITE: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeInsertRequest(this.channels[finalI], results, finalI + 1); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeInsertRequest(this.channels[finalI], results, finalI + 1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -219,16 +221,22 @@ private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, byte[] scratch = new byte[4 * 1024 * 1024]; for (int i = 0; i < args.calls; i++) { String object = objectResolver.Resolve(threadId, i); - ReadObjectRequest readRequest = ReadObjectRequest.newBuilder() - .setBucket(toV2BucketName(args.bkt)).setObject(object).build(); + ReadObjectRequest readRequest = + ReadObjectRequest.newBuilder() + .setBucket(toV2BucketName(args.bkt)) + .setObject(object) + .build(); long start = System.currentTimeMillis(); long totalBytes = 0; Iterator resIterator; if (useZeroCopy) { resIterator = - io.grpc.stub.ClientCalls.blockingServerStreamingCall(blockingStub.getChannel(), - readObjectMethod, blockingStub.getCallOptions(), readRequest); + io.grpc.stub.ClientCalls.blockingServerStreamingCall( + blockingStub.getChannel(), + readObjectMethod, + blockingStub.getCallOptions(), + readRequest); } else { resIterator = blockingStub.readObject(readRequest); } @@ -267,7 +275,7 @@ private void makeRandomReadRequest(ManagedChannel channel, ResultTable results, blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds)); } - String object = objectResolver.Resolve(threadId, 0); + String object = objectResolver.Resolve(threadId, /*objectId=*/ 0); ReadObjectRequest.Builder reqBuilder = ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(object); Random r = new Random(); @@ -361,13 +369,14 @@ public void onCompleted() { } } - private WriteObjectRequest getWriteRequest(boolean first, boolean last, int offset, - ByteString bytes, String obj) { + private WriteObjectRequest getWriteRequest( + boolean first, boolean last, int offset, ByteString bytes, String obj) { WriteObjectRequest.Builder builder = WriteObjectRequest.newBuilder(); if (first) { - builder.setWriteObjectSpec(WriteObjectSpec.newBuilder() - .setResource(Object.newBuilder().setBucket(toV2BucketName(args.bkt)).setName(obj)) - .build()); + builder.setWriteObjectSpec( + WriteObjectSpec.newBuilder() + .setResource(Object.newBuilder().setBucket(toV2BucketName(args.bkt)).setName(obj)) + .build()); } builder.setChecksummedData(ChecksummedData.newBuilder().setContent(bytes).build()); diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java index 8f4f06b3..118f7bf1 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/HttpClient.java @@ -34,13 +34,13 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep if (args.threads == 0) { switch (args.method) { case METHOD_READ: - makeMediaRequest(results, 1); + makeMediaRequest(results, /*threadId=*/ 1); break; case METHOD_RANDOM: - makeRandomMediaRequest(results, 1); + makeRandomMediaRequest(results, /*threadId=*/ 1); break; case METHOD_WRITE: - makeInsertRequest(results, 1); + makeInsertRequest(results, /*threadId=*/ 1); break; default: logger.warning("Please provide valid methods with --method"); @@ -59,13 +59,14 @@ public void startCalls(ResultTable results) throws InterruptedException, IOExcep case METHOD_RANDOM: for (int i = 0; i < args.threads; i++) { int finalI = i; - Runnable task = () -> { - try { - makeRandomMediaRequest(results, finalI + 1); - } catch (IOException e) { - e.printStackTrace(); - } - }; + Runnable task = + () -> { + try { + makeRandomMediaRequest(results, finalI + 1); + } catch (IOException e) { + e.printStackTrace(); + } + }; threadPoolExecutor.execute(task); } break; @@ -104,7 +105,7 @@ public void makeMediaRequest(ResultTable results, int threadId) { public void makeRandomMediaRequest(ResultTable results, int threadId) throws IOException { Random r = new Random(); - String object = objectResolver.Resolve(threadId, 0); + String object = objectResolver.Resolve(threadId, /*objectId=*/ 0); BlobId blobId = BlobId.of(args.bkt, object); ReadChannel reader = client.reader(blobId); for (int i = 0; i < args.calls; i++) { From 78b9895779eee20315263961895012365e0c25b7 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 11 May 2022 15:22:20 -0700 Subject: [PATCH 57/87] Added grpc-census & grpc-googleapis --- end2end-test-examples/gcs/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index 4a07dd4b..075cbc0b 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -24,7 +24,9 @@ dependencies { compile "io.grpc:grpc-alts:${grpcVersion}" compile "io.grpc:grpc-api:${grpcVersion}" compile "io.grpc:grpc-auth:${grpcVersion}" + compile "io.grpc:grpc-census:${grpcVersion}" compile "io.grpc:grpc-context:${grpcVersion}" + compile "io.grpc:grpc-googleapis:${grpcVersion}" compile "io.grpc:grpc-netty-shaded:${grpcVersion}" compile "io.grpc:grpc-protobuf:${grpcVersion}" compile "io.grpc:grpc-protobuf-lite:${grpcVersion}" From c943ccb33fe6d2c4db56794d27a4ebbd5cb24fc1 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 11 May 2022 17:59:35 -0700 Subject: [PATCH 58/87] Count fallbacks for calls without an affinity key. --- .../google/cloud/grpc/GcpManagedChannel.java | 18 +++++++++++---- .../cloud/grpc/GcpManagedChannelTest.java | 23 ++++++++++--------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index b3b5095c..4ddf58b5 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -784,7 +784,7 @@ public int getMaxActiveStreams() { */ protected ChannelRef getChannelRef(@Nullable String key) { if (key == null || key.isEmpty()) { - return pickLeastBusyChannel(); + return pickLeastBusyChannel(/* forFallback= */ false); } ChannelRef mappedChannel = affinityKeyToChannelRef.get(key); if (mappedChannel == null || !fallbackEnabled) { @@ -804,12 +804,14 @@ protected ChannelRef getChannelRef(@Nullable String key) { return channelRefs.get(channelId); } // No temp mapping for this key or fallback channel is also broken. - ChannelRef channelRef = pickLeastBusyChannel(); + ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ true); if (!fallbackMap.containsKey(channelRef.getId()) && channelRef.getActiveStreamsCount() < DEFAULT_MAX_STREAM) { // Got a ready and not an overloaded channel. - fallbacksSucceeded.incrementAndGet(); - tempMap.put(key, channelRef.getId()); + if (channelRef.getId() != mappedChannel.getId()) { + fallbacksSucceeded.incrementAndGet(); + tempMap.put(key, channelRef.getId()); + } return channelRef; } fallbacksFailed.incrementAndGet(); @@ -835,7 +837,7 @@ private synchronized ChannelRef createNewChannel() { * channel in the READY state and having fewer than maximum allowed number of active streams will * be provided if available. */ - private ChannelRef pickLeastBusyChannel() { + private ChannelRef pickLeastBusyChannel(boolean forFallback) { if (channelRefs.isEmpty()) { return createNewChannel(); } @@ -869,10 +871,16 @@ private ChannelRef pickLeastBusyChannel() { } if (channelRefs.size() < maxSize && readyMinStreams >= maxConcurrentStreamsLowWatermark) { + if (!forFallback) { + fallbacksSucceeded.incrementAndGet(); + } return createNewChannel(); } if (readyCandidate != null) { + if (!forFallback && readyCandidate.getId() != channelCandidate.getId()) { + fallbacksSucceeded.incrementAndGet(); + } return readyCandidate; } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 0635e538..3ba27c46 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -270,6 +270,8 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(1, chRef.getId()); assertEquals(2, pool.getNumberOfChannels()); + // This was a fallback from non-ready channel 0 to the newly created channel 1. + assertFallbacksMetric(fakeRegistry, 1, 0); // Adding one active stream to channel 1. pool.channelRefs.get(1).activeStreamsCountIncr(); @@ -279,6 +281,8 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(1, chRef.getId()); assertEquals(2, pool.getNumberOfChannels()); + // This was the second fallback from non-ready channel 0 to the channel 1. + assertFallbacksMetric(fakeRegistry, 2, 0); // Now let's have channel 0 still as not ready but bring channel 1 streams to low watermark. for (int i = 0; i < lowWatermark - 1; i++) { @@ -289,6 +293,8 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(2, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); + // This was the third fallback from non-ready channel 0 to the newly created channel 2. + assertFallbacksMetric(fakeRegistry, 3, 0); // Now we reached max pool size. Let's bring channel 2 to the low watermark and channel 1 to the // low watermark + 1 streams. @@ -305,6 +311,8 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(2, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); + // This was the fourth fallback from non-ready channel 0 to the channel 2. + assertFallbacksMetric(fakeRegistry, 4, 0); // Let's bring channel 1 to max streams and mark channel 2 as not ready. for (int i = 0; i < MAX_STREAM - lowWatermark; i++) { @@ -321,13 +329,6 @@ public void testGetChannelRefWithFallback() { assertEquals(0, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); - // So far the fallback logic sometimes provided different channels than a pool with disabled - // fallback would provide. But for metrics we consider a fallback only if we have an affinity - // key that was mapped to some channel and after that channel went to a non-ready state we - // temporarily used another channel as a fallback. - // Because of that, metric values for successful and failed fallbacks should be still zero. - assertFallbacksMetric(fakeRegistry, 0, 0); - // Let's have an affinity key and bind it to channel 0. final String key = "ABC"; pool.bind(pool.channelRefs.get(0), Collections.singletonList(key)); @@ -337,7 +338,7 @@ public void testGetChannelRefWithFallback() { // The getChannelRef should return the original channel 0 and report a failed fallback. chRef = pool.getChannelRef(key); assertEquals(0, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 0, 1); + assertFallbacksMetric(fakeRegistry, 4, 1); // Let's return channel 1 to a ready state. pool.processChannelStateChange(1, ConnectivityState.READY); @@ -345,7 +346,7 @@ public void testGetChannelRefWithFallback() { // The getChannelRef should return the channel 1 and report a successful fallback. chRef = pool.getChannelRef(key); assertEquals(1, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 1, 1); + assertFallbacksMetric(fakeRegistry, 5, 1); // Let's bring channel 1 back to connecting state. pool.processChannelStateChange(1, ConnectivityState.CONNECTING); @@ -354,7 +355,7 @@ public void testGetChannelRefWithFallback() { // The getChannelRef should return the channel 1 and report a failed fallback. chRef = pool.getChannelRef(key); assertEquals(1, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 1, 2); + assertFallbacksMetric(fakeRegistry, 5, 2); // Finally, we bring both channel 1 and channel 0 to the ready state and we should get the // original channel 0 for the key without any fallbacks happening. @@ -362,7 +363,7 @@ public void testGetChannelRefWithFallback() { pool.processChannelStateChange(0, ConnectivityState.READY); chRef = pool.getChannelRef(key); assertEquals(0, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 1, 2); + assertFallbacksMetric(fakeRegistry, 5, 2); } @Test From fc32319774e52b142ea2705eb704765dc022bdcf Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 14 May 2022 14:40:15 -0700 Subject: [PATCH 59/87] Support 4GiB for gcsio --- .../main/java/io/grpc/gcs/GcsioClient.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 8f87cfa5..66337208 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -143,22 +143,25 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept new GoogleCloudStorageFileSystem( creds, GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); - - int size = args.size * 1024; - ByteBuffer buff = ByteBuffer.allocate(size); + long totalSize = args.size * 1024L; + long receivedSize = 0; + int buffSize = (args.buffSize == 0 ? 32 * 1024 : args.buffSize) * 1024; + ByteBuffer buff = ByteBuffer.allocate(buffSize); for (int i = 0; i < args.calls; i++) { long start = System.currentTimeMillis(); String object = objectResolver.Resolve(threadId, i); URI uri = URI.create("gs://" + args.bkt + "/" + object); ReadableByteChannel readChannel = gcsfs.open(uri); - readChannel.read(buff); - long dur = System.currentTimeMillis() - start; - if (buff.remaining() > 0) { - logger.warning("Got remaining bytes: " + buff.remaining()); + while (receivedSize < totalSize) { + int r = readChannel.read(buff); + if (r < 0) break; + buff.rewind(); + receivedSize += r; } + long dur = System.currentTimeMillis() - start; buff.clear(); readChannel.close(); - results.reportResult(args.bkt, object, size, dur); + results.reportResult(args.bkt, object, receivedSize, dur); } gcsfs.close(); From 019386d81187200a099139c989ea2fd2fe29d1c2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 14 May 2022 17:44:13 -0700 Subject: [PATCH 60/87] Fix --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 66337208..dddf945a 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -155,7 +155,7 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept while (receivedSize < totalSize) { int r = readChannel.read(buff); if (r < 0) break; - buff.rewind(); + buff.clear(); receivedSize += r; } long dur = System.currentTimeMillis() - start; From 272769a9c561f4f32e1b3c1236936f75d22847d6 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 14 May 2022 17:45:16 -0700 Subject: [PATCH 61/87] Added log --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index dddf945a..185662e1 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -154,6 +154,7 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept ReadableByteChannel readChannel = gcsfs.open(uri); while (receivedSize < totalSize) { int r = readChannel.read(buff); + logger.warning("R: " + r); if (r < 0) break; buff.clear(); receivedSize += r; From d192bbed6a6e3e592a61db9bac18fdded527c91c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 14 May 2022 17:47:04 -0700 Subject: [PATCH 62/87] Fix bug --- .../gcs/src/main/java/io/grpc/gcs/GcsioClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java index 185662e1..606c5658 100644 --- a/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java +++ b/end2end-test-examples/gcs/src/main/java/io/grpc/gcs/GcsioClient.java @@ -144,17 +144,16 @@ private void makeMediaRequest(ResultTable results, int threadId) throws IOExcept creds, GoogleCloudStorageFileSystemOptions.builder().setCloudStorageOptions(gcsOpts).build()); long totalSize = args.size * 1024L; - long receivedSize = 0; int buffSize = (args.buffSize == 0 ? 32 * 1024 : args.buffSize) * 1024; ByteBuffer buff = ByteBuffer.allocate(buffSize); for (int i = 0; i < args.calls; i++) { + long receivedSize = 0; long start = System.currentTimeMillis(); String object = objectResolver.Resolve(threadId, i); URI uri = URI.create("gs://" + args.bkt + "/" + object); ReadableByteChannel readChannel = gcsfs.open(uri); while (receivedSize < totalSize) { int r = readChannel.read(buff); - logger.warning("R: " + r); if (r < 0) break; buff.clear(); receivedSize += r; From efe135f2c1dd6fb7d9c35dec2f40093af68bbde0 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 18 May 2022 10:53:25 -0700 Subject: [PATCH 63/87] Log metrics with fine log level. --- .../google/cloud/grpc/GcpManagedChannel.java | 138 ++++++++++++++++-- .../cloud/grpc/GcpManagedChannelTest.java | 121 ++++++++++++++- 2 files changed, 244 insertions(+), 15 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 2597e193..28b9dc2e 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -17,6 +17,7 @@ package com.google.cloud.grpc; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions; @@ -50,6 +51,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -100,6 +103,10 @@ public class GcpManagedChannel extends ManagedChannel { new ArrayList<>( Collections.singletonList(LabelValue.create(GcpMetricsConstants.RESULT_ERROR))); private String metricPrefix; + private final String metricPoolIndex = + String.format("pool-%d", channelPoolIndex.incrementAndGet()); + private final Map cumulativeMetricValues = new ConcurrentHashMap<>(); + private ScheduledExecutorService logMetricService; // Metrics counters. private final AtomicInteger readyChannels = new AtomicInteger(); @@ -198,14 +205,24 @@ private void initOptions() { initMetrics(); } + private synchronized void initLogMetrics() { + if (logMetricService != null) { + return; + } + logMetricService = Executors.newSingleThreadScheduledExecutor(); + logMetricService.scheduleAtFixedRate(this::logMetrics, 60, 60, SECONDS); + } + private void initMetrics() { final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); if (metricsOptions == null) { logger.info("Metrics options are empty. Metrics disabled."); + initLogMetrics(); return; } if (metricsOptions.getMetricRegistry() == null) { logger.info("Metric registry is null. Metrics disabled."); + initLogMetrics(); return; } logger.info("Metrics enabled."); @@ -221,8 +238,7 @@ private void initMetrics() { LabelKey.create(GcpMetricsConstants.POOL_INDEX_LABEL, GcpMetricsConstants.POOL_INDEX_DESC); labelKeys.add(poolKey); labelKeysWithResult.add(poolKey); - final LabelValue poolIndex = - LabelValue.create(String.format("pool-%d", channelPoolIndex.incrementAndGet())); + final LabelValue poolIndex = LabelValue.create(metricPoolIndex); labelValues.add(poolIndex); labelValuesSuccess.add(poolIndex); labelValuesError.add(poolIndex); @@ -409,6 +425,50 @@ private void initMetrics() { GcpManagedChannel::reportMaxUnresponsiveDrops); } + private void logGauge(String key, long value) { + logger.fine(String.format("%s stat: %s = %d", metricPoolIndex, key, value)); + } + + private void logCumulative(String key, long value) { + logger.fine(() -> { + Long prevValue = cumulativeMetricValues.put(key, value); + long logValue = prevValue == null ? value : value - prevValue; + return String.format("%s stat: %s = %d", metricPoolIndex, key, logValue); + }); + } + + private void logMetrics() { + reportMinReadyChannels(); + reportMaxReadyChannels(); + reportMaxChannels(); + reportMaxAllowedChannels(); + reportNumChannelDisconnect(); + reportNumChannelConnect(); + reportMinReadinessTime(); + reportAvgReadinessTime(); + reportMaxReadinessTime(); + reportMinActiveStreams(); + reportMaxActiveStreams(); + reportMinTotalActiveStreams(); + reportMaxTotalActiveStreams(); + reportMinAffinity(); + reportMaxAffinity(); + reportNumAffinity(); + reportMinOkCalls(); + reportMinErrCalls(); + reportMaxOkCalls(); + reportMaxErrCalls(); + reportTotalOkCalls(); + reportTotalErrCalls(); + reportSucceededFallbacks(); + reportFailedFallbacks(); + reportUnresponsiveDetectionCount(); + reportMinUnresponsiveMs(); + reportMaxUnresponsiveMs(); + reportMinUnresponsiveDrops(); + reportMaxUnresponsiveDrops(); + } + private MetricOptions createMetricOptions( String description, List labelKeys, String unit) { return MetricOptions.builder() @@ -472,37 +532,48 @@ private void createDerivedLongCumulativeTimeSeriesWithResult( metric.createTimeSeries(labelValuesError, obj, funcErr); } + // TODO: When introducing pool downscaling feature this method must be changed accordingly. private long reportMaxChannels() { - return getNumberOfChannels(); + int value = getNumberOfChannels(); + logGauge(GcpMetricsConstants.METRIC_MAX_CHANNELS, value); + return value; } private long reportMaxAllowedChannels() { + logGauge(GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS, maxSize); return maxSize; } private long reportMinReadyChannels() { int value = minReadyChannels; minReadyChannels = readyChannels.get(); + logGauge(GcpMetricsConstants.METRIC_MIN_READY_CHANNELS, value); return value; } private long reportMaxReadyChannels() { int value = maxReadyChannels; maxReadyChannels = readyChannels.get(); + logGauge(GcpMetricsConstants.METRIC_MAX_READY_CHANNELS, value); return value; } private long reportNumChannelConnect() { - return numChannelConnect.get(); + long value = numChannelConnect.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_CHANNEL_CONNECT, value); + return value; } private long reportNumChannelDisconnect() { - return numChannelDisconnect.get(); + long value = numChannelDisconnect.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_CHANNEL_DISCONNECT, value); + return value; } private long reportMinReadinessTime() { long value = minReadinessTime; minReadinessTime = 0; + logGauge(GcpMetricsConstants.METRIC_MIN_CHANNEL_READINESS_TIME, value); return value; } @@ -513,12 +584,14 @@ private long reportAvgReadinessTime() { if (occ != 0) { value = total / occ; } + logGauge(GcpMetricsConstants.METRIC_AVG_CHANNEL_READINESS_TIME, value); return value; } private long reportMaxReadinessTime() { long value = maxReadinessTime; maxReadinessTime = 0; + logGauge(GcpMetricsConstants.METRIC_MAX_CHANNEL_READINESS_TIME, value); return value; } @@ -526,6 +599,7 @@ private int reportMinActiveStreams() { int value = minActiveStreams; minActiveStreams = channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).min().orElse(0); + logGauge(GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS, value); return value; } @@ -533,51 +607,62 @@ private int reportMaxActiveStreams() { int value = maxActiveStreams; maxActiveStreams = channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).max().orElse(0); + logGauge(GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS, value); return value; } private int reportMinTotalActiveStreams() { int value = minTotalActiveStreams; minTotalActiveStreams = totalActiveStreams.get(); + logGauge(GcpMetricsConstants.METRIC_MIN_TOTAL_ACTIVE_STREAMS, value); return value; } private int reportMaxTotalActiveStreams() { int value = maxTotalActiveStreams; maxTotalActiveStreams = totalActiveStreams.get(); + logGauge(GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS, value); return value; } private int reportMinAffinity() { int value = minAffinity; minAffinity = channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).min().orElse(0); + logGauge(GcpMetricsConstants.METRIC_MIN_AFFINITY, value); return value; } private int reportMaxAffinity() { int value = maxAffinity; maxAffinity = channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).max().orElse(0); + logGauge(GcpMetricsConstants.METRIC_MAX_AFFINITY, value); return value; } private int reportNumAffinity() { - return totalAffinityCount.get(); + int value = totalAffinityCount.get(); + logGauge(GcpMetricsConstants.METRIC_NUM_AFFINITY, value); + return value; } private synchronized long reportMinOkCalls() { minOkReported = true; calcMinMaxOkCalls(); + logGauge(GcpMetricsConstants.METRIC_MIN_CALLS + "_ok", minOkCalls); return minOkCalls; } private synchronized long reportMaxOkCalls() { maxOkReported = true; calcMinMaxOkCalls(); + logGauge(GcpMetricsConstants.METRIC_MAX_CALLS + "_ok", maxOkCalls); return maxOkCalls; } private long reportTotalOkCalls() { - return totalOkCalls.get(); + long value = totalOkCalls.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_CALLS_COMPLETED + "_ok", value); + return value; } private void calcMinMaxOkCalls() { @@ -595,17 +680,21 @@ private void calcMinMaxOkCalls() { private synchronized long reportMinErrCalls() { minErrReported = true; calcMinMaxErrCalls(); + logGauge(GcpMetricsConstants.METRIC_MIN_CALLS + "_err", minErrCalls); return minErrCalls; } private synchronized long reportMaxErrCalls() { maxErrReported = true; calcMinMaxErrCalls(); + logGauge(GcpMetricsConstants.METRIC_MAX_CALLS + "_err", maxErrCalls); return maxErrCalls; } private long reportTotalErrCalls() { - return totalErrCalls.get(); + long value = totalErrCalls.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_CALLS_COMPLETED + "_err", value); + return value; } private void calcMinMaxErrCalls() { @@ -621,38 +710,48 @@ private void calcMinMaxErrCalls() { } private long reportSucceededFallbacks() { - return fallbacksSucceeded.get(); + long value = fallbacksSucceeded.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_FALLBACKS + "_ok", value); + return value; } private long reportFailedFallbacks() { - return fallbacksFailed.get(); + long value = fallbacksFailed.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_FALLBACKS + "_fail", value); + return value; } private long reportUnresponsiveDetectionCount() { - return unresponsiveDetectionCount.get(); + long value = unresponsiveDetectionCount.get(); + logCumulative(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS, value); + return value; } private long reportMinUnresponsiveMs() { long value = minUnresponsiveMs; minUnresponsiveMs = 0; + logGauge(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME, value); return value; } private long reportMaxUnresponsiveMs() { long value = maxUnresponsiveMs; maxUnresponsiveMs = 0; + logGauge(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME, value); return value; } private long reportMinUnresponsiveDrops() { long value = minUnresponsiveDrops; minUnresponsiveDrops = 0; + logGauge(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS, value); return value; } private long reportMaxUnresponsiveDrops() { long value = maxUnresponsiveDrops; maxUnresponsiveDrops = 0; + logGauge(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS, value); return value; } @@ -958,6 +1057,9 @@ public ManagedChannel shutdownNow() { channelRef.getChannel().shutdownNow(); } } + if (logMetricService != null && !logMetricService.isTerminated()) { + logMetricService.shutdownNow(); + } return this; } @@ -966,6 +1068,9 @@ public ManagedChannel shutdown() { for (ChannelRef channelRef : channelRefs) { channelRef.getChannel().shutdown(); } + if (logMetricService != null) { + logMetricService.shutdown(); + } return this; } @@ -982,6 +1087,11 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE } channelRef.getChannel().awaitTermination(awaitTimeNanos, NANOSECONDS); } + long awaitTimeNanos = endTimeNanos - System.nanoTime(); + if (logMetricService != null && awaitTimeNanos > 0) { + //noinspection ResultOfMethodCallIgnored + logMetricService.awaitTermination(awaitTimeNanos, NANOSECONDS); + } return isTerminated(); } @@ -992,6 +1102,9 @@ public boolean isShutdown() { return false; } } + if (logMetricService != null) { + return logMetricService.isShutdown(); + } return true; } @@ -1002,6 +1115,9 @@ public boolean isTerminated() { return false; } } + if (logMetricService != null) { + return logMetricService.isTerminated(); + } return true; } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 3ba27c46..2b6f5b13 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -50,9 +50,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -71,6 +76,30 @@ public final class GcpManagedChannelTest { private static final int MAX_CHANNEL = 10; private static final int MAX_STREAM = 100; + private static final Logger testLogger = Logger.getLogger(GcpManagedChannel.class.getName()); + + private final List logRecords = new LinkedList<>(); + + private String lastLogMessage() { + return logRecords.get(logRecords.size() - 1).getMessage(); + } + + private Level lastLogLevel() { + return logRecords.get(logRecords.size() - 1).getLevel(); + } + + private final Handler testLogHandler = new Handler() { + @Override + public void publish(LogRecord record) { + logRecords.add(record); + } + + @Override + public void flush() {} + + @Override + public void close() throws SecurityException {} + }; private GcpManagedChannel gcpChannel; private ManagedChannelBuilder builder; @@ -83,6 +112,7 @@ private void resetGcpChannel() { @Before public void setUpChannel() { + testLogger.addHandler(testLogHandler); builder = ManagedChannelBuilder.forAddress(TARGET, 443); gcpChannel = (GcpManagedChannel) GcpManagedChannelBuilder.forDelegateBuilder(builder).build(); } @@ -90,6 +120,9 @@ public void setUpChannel() { @After public void shutdown() { gcpChannel.shutdownNow(); + testLogger.removeHandler(testLogHandler); + testLogger.setLevel(Level.INFO); + logRecords.clear(); } @Test @@ -530,6 +563,8 @@ public void testParseEmptyChannelJsonFile() { @Test public void testMetrics() { + // Watch debug messages. + testLogger.setLevel(Level.FINE); final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry(); final String prefix = "some/prefix/"; final List labelKeys = @@ -562,7 +597,8 @@ public void testMetrics() { LabelKey.create(GcpMetricsConstants.POOL_INDEX_LABEL, GcpMetricsConstants.POOL_INDEX_DESC)); List expectedLabelValues = new ArrayList<>(labelValues); int currentIndex = GcpManagedChannel.channelPoolIndex.get(); - expectedLabelValues.add(LabelValue.create(String.format("pool-%d", currentIndex))); + String poolIndex = String.format("pool-%d", currentIndex); + expectedLabelValues.add(LabelValue.create(poolIndex)); try { // Let's fill five channels with some fake streams. @@ -577,12 +613,19 @@ public void testMetrics() { MetricsRecord record = fakeRegistry.pollRecord(); assertThat(record.getMetrics().size()).isEqualTo(25); + // Initial log messages count. + int logCount = logRecords.size(); + List> numChannels = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_CHANNELS); assertThat(numChannels.size()).isEqualTo(1); assertThat(numChannels.get(0).value()).isEqualTo(5L); assertThat(numChannels.get(0).keys()).isEqualTo(expectedLabelKeys); assertThat(numChannels.get(0).values()).isEqualTo(expectedLabelValues); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_CHANNELS + " = 5"); List> maxAllowedChannels = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS); @@ -590,6 +633,10 @@ public void testMetrics() { assertThat(maxAllowedChannels.get(0).value()).isEqualTo(MAX_CHANNEL); assertThat(maxAllowedChannels.get(0).keys()).isEqualTo(expectedLabelKeys); assertThat(maxAllowedChannels.get(0).values()).isEqualTo(expectedLabelValues); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS + " = 10"); List> minActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS); @@ -597,6 +644,10 @@ public void testMetrics() { assertThat(minActiveStreams.get(0).value()).isEqualTo(0L); assertThat(minActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys); assertThat(minActiveStreams.get(0).values()).isEqualTo(expectedLabelValues); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS + " = 0"); List> maxActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS); @@ -604,14 +655,23 @@ public void testMetrics() { assertThat(maxActiveStreams.get(0).value()).isEqualTo(7L); assertThat(maxActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys); assertThat(maxActiveStreams.get(0).values()).isEqualTo(expectedLabelValues); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS + " = 7"); List> totalActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS); assertThat(totalActiveStreams.size()).isEqualTo(1); - assertThat(totalActiveStreams.get(0).value()) - .isEqualTo(Arrays.stream(streams).asLongStream().sum()); + long totalStreamsExpected = Arrays.stream(streams).asLongStream().sum(); + assertThat(totalActiveStreams.get(0).value()).isEqualTo(totalStreamsExpected); assertThat(totalActiveStreams.get(0).keys()).isEqualTo(expectedLabelKeys); assertThat(totalActiveStreams.get(0).values()).isEqualTo(expectedLabelValues); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS + " = " + + totalStreamsExpected); } finally { pool.shutdownNow(); } @@ -619,6 +679,8 @@ public void testMetrics() { @Test public void testUnresponsiveDetection() throws InterruptedException { + // Watch debug messages. + testLogger.setLevel(Level.FINE); final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry(); // Creating a pool with unresponsive connection detection for 100 ms, 3 dropped requests. final GcpManagedChannel pool = @@ -634,6 +696,8 @@ public void testUnresponsiveDetection() throws InterruptedException { GcpMetricsOptions.newBuilder().withMetricRegistry(fakeRegistry).build()) .build()) .build(); + int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + String poolIndex = String.format("pool-%d", currentIndex); final AtomicInteger idleCounter = new AtomicInteger(); ManagedChannel channel = new FakeIdleCountingManagedChannel(idleCounter); ChannelRef chRef = pool.new ChannelRef(channel, 0); @@ -652,23 +716,52 @@ public void testUnresponsiveDetection() throws InterruptedException { // Reconnected after 3rd deadline exceeded. assertEquals(1, idleCounter.get()); + // Initial log messages count. + int logCount = logRecords.size(); + MetricsRecord record = fakeRegistry.pollRecord(); List> metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isEqualTo(1L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isEqualTo(3L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS + " = 3"); + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isEqualTo(3L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS + " = 3"); + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isAtLeast(100L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).matches( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME + + " = 1\\d\\d"); + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isAtLeast(100L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + assertThat(lastLogMessage()).matches( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME + + " = 1\\d\\d"); // Any message from the server must reset the dropped requests count and timestamp. TimeUnit.MILLISECONDS.sleep(105); @@ -708,9 +801,29 @@ public void testUnresponsiveDetection() throws InterruptedException { assertEquals(1, idleCounter.get()); TimeUnit.MILLISECONDS.sleep(105); - // Any subsequent deadline exceeded after 100ms must trigger the reconnect. + // Any subsequent deadline exceeded after 100ms must trigger the reconnection. chRef.activeStreamsCountDecr(startNanos, deStatus, false); assertEquals(2, idleCounter.get()); + + // The cumulative num_unresponsive_detections metric must become 2. + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); + assertThat(metric.size()).isEqualTo(1); + assertThat(metric.get(0).value()).isEqualTo(2L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + // But the log metric count the detections since previous report for num_unresponsive_detections + // in the logs. It is always delta in the logs, not cumulative. + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); + // If we log it again the cumulative metric value must remain unchanged. + metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); + assertThat(metric.size()).isEqualTo(1); + assertThat(metric.get(0).value()).isEqualTo(2L); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); + // But in the log it must post 0. + assertThat(lastLogMessage()).isEqualTo( + poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 0"); } static class FakeIdleCountingManagedChannel extends ManagedChannel { From 91d8573b8f4003dba1f21506e0b70ef425570858 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 18 May 2022 16:26:55 -0700 Subject: [PATCH 64/87] Bind keys on the fly. Previously, if GcpManagedChannel didn't get a key from a bind call, then it will always return the least busy channel for the key effectively providing no affinity. But it is not guaranteed that GcpManagedChannel will have a key from a bind call. E.g., an affinity key may be obtained by the upstream by other means. After this change a key will be mapped to a channel on a bind call or on the first use. --- .../google/cloud/grpc/GcpManagedChannel.java | 7 +++++- .../cloud/grpc/GcpManagedChannelTest.java | 25 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 2597e193..91baea78 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -822,7 +822,12 @@ protected ChannelRef getChannelRef(@Nullable String key) { return pickLeastBusyChannel(/* forFallback= */ false); } ChannelRef mappedChannel = affinityKeyToChannelRef.get(key); - if (mappedChannel == null || !fallbackEnabled) { + if (mappedChannel == null) { + ChannelRef channelRef = pickLeastBusyChannel(/*forFallback= */ false); + bind(channelRef, Collections.singletonList(key)); + return channelRef; + } + if (!fallbackEnabled) { return mappedChannel; } // Look up if the channelRef is not ready. diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 3ba27c46..d1d15be9 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -382,7 +382,7 @@ public void testGetChannelRefMaxSize() { public void testBindUnbindKey() { // Initialize the channel and bind the key, check the affinity count. ChannelRef cf1 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 5); - ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 4); + ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 2, 0, 4); gcpChannel.channelRefs.add(cf1); gcpChannel.channelRefs.add(cf2); gcpChannel.bind(cf1, Collections.singletonList("key1")); @@ -418,6 +418,29 @@ public void testBindUnbindKey() { assertEquals(0, gcpChannel.affinityKeyToChannelRef.size()); } + @Test + public void testUsingKeyWithoutBinding() { + // Initialize the channel and bind the key, check the affinity count. + ChannelRef cf1 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 5); + ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 2, 0, 4); + gcpChannel.channelRefs.add(cf1); + gcpChannel.channelRefs.add(cf2); + + final String key = "non-binded-key"; + ChannelRef channelRef = gcpChannel.getChannelRef(key); + // Should bind on the fly to the least busy channel, which is 2. + assertThat(channelRef.getId()).isEqualTo(2); + + cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true); + cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true); + // Even after channel 1 now has less active streams (3) the channel 2 is still mapped for the + // same key. + + channelRef = gcpChannel.getChannelRef(key); + // Should bind on the fly to the least busy channel, which is 2. + assertThat(channelRef.getId()).isEqualTo(2); + } + @Test public void testGetKeysFromRequest() { String expected = "thisisaname"; From d960e84495432a3fa3c00d8a6eb82d8590a4c587 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Mon, 23 May 2022 11:41:24 -0700 Subject: [PATCH 65/87] Add pool index to all logs. --- .../google/cloud/grpc/GcpManagedChannel.java | 23 ++++++++++++------ .../cloud/grpc/GcpManagedChannelTest.java | 24 +++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 28b9dc2e..188ae7bd 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -56,6 +56,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -196,6 +197,14 @@ public GcpManagedChannel( } } + private Supplier log(Supplier messageSupplier) { + return () -> String.format("%s: %s", metricPoolIndex, messageSupplier.get()); + } + + private String log(String message) { + return String.format("%s: %s", metricPoolIndex, message); + } + private void initOptions() { GcpManagedChannelOptions.GcpChannelPoolOptions poolOptions = options.getChannelPoolOptions(); if (poolOptions != null) { @@ -216,16 +225,16 @@ private synchronized void initLogMetrics() { private void initMetrics() { final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); if (metricsOptions == null) { - logger.info("Metrics options are empty. Metrics disabled."); + logger.info(log("Metrics options are empty. Metrics disabled.")); initLogMetrics(); return; } if (metricsOptions.getMetricRegistry() == null) { - logger.info("Metric registry is null. Metrics disabled."); + logger.info(log("Metric registry is null. Metrics disabled.")); initLogMetrics(); return; } - logger.info("Metrics enabled."); + logger.info(log("Metrics enabled.")); metricRegistry = metricsOptions.getMetricRegistry(); labelKeys.addAll(metricsOptions.getLabelKeys()); @@ -426,15 +435,15 @@ private void initMetrics() { } private void logGauge(String key, long value) { - logger.fine(String.format("%s stat: %s = %d", metricPoolIndex, key, value)); + logger.fine(log(String.format("stat: %s = %d", key, value))); } private void logCumulative(String key, long value) { - logger.fine(() -> { + logger.fine(log(() -> { Long prevValue = cumulativeMetricValues.put(key, value); long logValue = prevValue == null ? value : value - prevValue; - return String.format("%s stat: %s = %d", metricPoolIndex, key, logValue); - }); + return String.format("stat: %s = %d", key, logValue); + })); } private void logMetrics() { diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 2b6f5b13..9bb080f2 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -625,7 +625,7 @@ public void testMetrics() { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_CHANNELS + " = 5"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_CHANNELS + " = 5"); List> maxAllowedChannels = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS); @@ -636,7 +636,7 @@ public void testMetrics() { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS + " = 10"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_ALLOWED_CHANNELS + " = 10"); List> minActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS); @@ -647,7 +647,7 @@ public void testMetrics() { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS + " = 0"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MIN_ACTIVE_STREAMS + " = 0"); List> maxActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS); @@ -658,7 +658,7 @@ public void testMetrics() { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS + " = 7"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_ACTIVE_STREAMS + " = 7"); List> totalActiveStreams = record.getMetrics().get(prefix + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS); @@ -670,7 +670,7 @@ public void testMetrics() { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS + " = " + + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_TOTAL_ACTIVE_STREAMS + " = " + totalStreamsExpected); } finally { pool.shutdownNow(); @@ -727,7 +727,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS); assertThat(metric.size()).isEqualTo(1); @@ -735,7 +735,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS + " = 3"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DROPPED_CALLS + " = 3"); metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS); assertThat(metric.size()).isEqualTo(1); @@ -743,7 +743,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS + " = 3"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DROPPED_CALLS + " = 3"); metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME); assertThat(metric.size()).isEqualTo(1); @@ -751,7 +751,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).matches( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME + + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MIN_UNRESPONSIVE_DETECTION_TIME + " = 1\\d\\d"); metric = record.getMetrics().get(GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME); @@ -760,7 +760,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(logRecords.size()).isEqualTo(++logCount); assertThat(lastLogLevel()).isEqualTo(Level.FINE); assertThat(lastLogMessage()).matches( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME + + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_MAX_UNRESPONSIVE_DETECTION_TIME + " = 1\\d\\d"); // Any message from the server must reset the dropped requests count and timestamp. @@ -814,7 +814,7 @@ public void testUnresponsiveDetection() throws InterruptedException { // But the log metric count the detections since previous report for num_unresponsive_detections // in the logs. It is always delta in the logs, not cumulative. assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); // If we log it again the cumulative metric value must remain unchanged. metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); assertThat(metric.size()).isEqualTo(1); @@ -823,7 +823,7 @@ public void testUnresponsiveDetection() throws InterruptedException { assertThat(lastLogLevel()).isEqualTo(Level.FINE); // But in the log it must post 0. assertThat(lastLogMessage()).isEqualTo( - poolIndex + " stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 0"); + poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 0"); } static class FakeIdleCountingManagedChannel extends ManagedChannel { From 5df87daae84ecdbdec95289a120d13eddcdc9343 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Mon, 23 May 2022 14:09:21 -0700 Subject: [PATCH 66/87] Log metric options. --- .../google/cloud/grpc/GcpManagedChannel.java | 33 +++++++++++++++++++ .../cloud/grpc/GcpManagedChannelTest.java | 14 ++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 188ae7bd..ab75fc4d 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; @@ -108,6 +109,7 @@ public class GcpManagedChannel extends ManagedChannel { String.format("pool-%d", channelPoolIndex.incrementAndGet()); private final Map cumulativeMetricValues = new ConcurrentHashMap<>(); private ScheduledExecutorService logMetricService; + private String metricsOptionsToLog; // Metrics counters. private final AtomicInteger readyChannels = new AtomicInteger(); @@ -222,6 +224,35 @@ private synchronized void initLogMetrics() { logMetricService.scheduleAtFixedRate(this::logMetrics, 60, 60, SECONDS); } + private void logMetricsOptions() { + if (metricsOptionsToLog != null) { + logger.fine(log(metricsOptionsToLog)); + return; + } + final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); + if (metricsOptions == null) { + return; + } + + Iterator keyIterator = metricsOptions.getLabelKeys().iterator(); + Iterator valueIterator = metricsOptions.getLabelValues().iterator(); + + final List tags = new ArrayList<>(); + while (keyIterator.hasNext() && valueIterator.hasNext()) { + tags.add( + String.format("%s = %s", keyIterator.next().getKey(), valueIterator.next().getValue()) + ); + } + + metricsOptionsToLog = String.format( + "Metrics name prefix = \"%s\", tags: %s", + metricsOptions.getNamePrefix(), + String.join(", ", tags) + ); + + logger.fine(log(metricsOptionsToLog)); + } + private void initMetrics() { final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); if (metricsOptions == null) { @@ -229,6 +260,7 @@ private void initMetrics() { initLogMetrics(); return; } + logMetricsOptions(); if (metricsOptions.getMetricRegistry() == null) { logger.info(log("Metric registry is null. Metrics disabled.")); initLogMetrics(); @@ -447,6 +479,7 @@ private void logCumulative(String key, long value) { } private void logMetrics() { + logMetricsOptions(); reportMinReadyChannels(); reportMaxReadyChannels(); reportMaxChannels(); diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 9bb080f2..edde1581 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -592,12 +592,22 @@ public void testMetrics() { .build()) .build(); + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String poolIndex = String.format("pool-%d", currentIndex); + + // Logs metrics options. + assertThat(logRecords.get(logRecords.size() - 2).getLevel()).isEqualTo(Level.FINE); + assertThat(logRecords.get(logRecords.size() - 2).getMessage()).isEqualTo( + poolIndex + ": Metrics name prefix = \"some/prefix/\", tags: key_a = val_a, key_b = val_b" + ); + + assertThat(lastLogLevel()).isEqualTo(Level.INFO); + assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Metrics enabled."); + List expectedLabelKeys = new ArrayList<>(labelKeys); expectedLabelKeys.add( LabelKey.create(GcpMetricsConstants.POOL_INDEX_LABEL, GcpMetricsConstants.POOL_INDEX_DESC)); List expectedLabelValues = new ArrayList<>(labelValues); - int currentIndex = GcpManagedChannel.channelPoolIndex.get(); - String poolIndex = String.format("pool-%d", currentIndex); expectedLabelValues.add(LabelValue.create(poolIndex)); try { From 712560af049dd18e3a68f05533d5745a65b0b32b Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Tue, 24 May 2022 13:55:36 -0700 Subject: [PATCH 67/87] fixed test comments --- .../java/com/google/cloud/grpc/GcpManagedChannelTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index d1d15be9..29209a86 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -433,11 +433,9 @@ public void testUsingKeyWithoutBinding() { cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true); cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true); + channelRef = gcpChannel.getChannelRef(key); // Even after channel 1 now has less active streams (3) the channel 2 is still mapped for the // same key. - - channelRef = gcpChannel.getChannelRef(key); - // Should bind on the fly to the least busy channel, which is 2. assertThat(channelRef.getId()).isEqualTo(2); } From f05d15a50d22d5bfe4479a52b4592ccbdeb02773 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Tue, 24 May 2022 10:53:42 -0700 Subject: [PATCH 68/87] Log creation and shutdown. --- .../google/cloud/grpc/GcpManagedChannel.java | 44 ++++++--------- .../cloud/grpc/GcpManagedChannelOptions.java | 54 +++++++++++++++++++ .../cloud/grpc/GcpManagedChannelTest.java | 5 +- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 799407b3..d895b84a 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -28,6 +28,7 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.TextFormat; import io.grpc.CallOptions; import io.grpc.ClientCall; import io.grpc.ConnectivityState; @@ -46,7 +47,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; @@ -109,7 +109,6 @@ public class GcpManagedChannel extends ManagedChannel { String.format("pool-%d", channelPoolIndex.incrementAndGet()); private final Map cumulativeMetricValues = new ConcurrentHashMap<>(); private ScheduledExecutorService logMetricService; - private String metricsOptionsToLog; // Metrics counters. private final AtomicInteger readyChannels = new AtomicInteger(); @@ -161,6 +160,11 @@ public GcpManagedChannel( loadApiConfig(apiConfig); this.delegateChannelBuilder = delegateChannelBuilder; this.options = options; + logger.finer(log( + "Created with api config: %s, and options: %s", + apiConfig == null ? "null" : TextFormat.shortDebugString(apiConfig), + options + )); initOptions(); if (options.getResiliencyOptions() != null) { fallbackEnabled = options.getResiliencyOptions().isNotReadyFallbackEnabled(); @@ -195,6 +199,7 @@ public GcpManagedChannel( GcpManagedChannelOptions options) { this(delegateChannelBuilder, apiConfig, options); if (poolSize != 0) { + logger.finer(log("Pool size adjusted to %d", poolSize)); this.maxSize = poolSize; } } @@ -207,6 +212,10 @@ private String log(String message) { return String.format("%s: %s", metricPoolIndex, message); } + private String log(String format, Object... args) { + return String.format("%s: %s", metricPoolIndex, String.format(format, args)); + } + private void initOptions() { GcpManagedChannelOptions.GcpChannelPoolOptions poolOptions = options.getChannelPoolOptions(); if (poolOptions != null) { @@ -225,32 +234,9 @@ private synchronized void initLogMetrics() { } private void logMetricsOptions() { - if (metricsOptionsToLog != null) { - logger.fine(log(metricsOptionsToLog)); - return; - } - final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); - if (metricsOptions == null) { - return; - } - - Iterator keyIterator = metricsOptions.getLabelKeys().iterator(); - Iterator valueIterator = metricsOptions.getLabelValues().iterator(); - - final List tags = new ArrayList<>(); - while (keyIterator.hasNext() && valueIterator.hasNext()) { - tags.add( - String.format("%s = %s", keyIterator.next().getKey(), valueIterator.next().getValue()) - ); + if (options.getMetricsOptions() != null) { + logger.fine(log("Metrics options: %s", options.getMetricsOptions())); } - - metricsOptionsToLog = String.format( - "Metrics name prefix = \"%s\", tags: %s", - metricsOptions.getNamePrefix(), - String.join(", ", tags) - ); - - logger.fine(log(metricsOptionsToLog)); } private void initMetrics() { @@ -467,7 +453,7 @@ private void initMetrics() { } private void logGauge(String key, long value) { - logger.fine(log(String.format("stat: %s = %d", key, value))); + logger.fine(log("stat: %s = %d", key, value)); } private void logCumulative(String key, long value) { @@ -1099,6 +1085,7 @@ public ClientCall newCall( @Override public ManagedChannel shutdownNow() { + logger.finer(log("Shutdown now started.")); for (ChannelRef channelRef : channelRefs) { if (!channelRef.getChannel().isTerminated()) { channelRef.getChannel().shutdownNow(); @@ -1112,6 +1099,7 @@ public ManagedChannel shutdownNow() { @Override public ManagedChannel shutdown() { + logger.finer(log("Shutdown started.")); for (ChannelRef channelRef : channelRefs) { channelRef.getChannel().shutdown(); } diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java index 9d5fe2d1..8684bd4a 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java @@ -21,6 +21,7 @@ import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.MetricRegistry; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -60,6 +61,16 @@ public GcpResiliencyOptions getResiliencyOptions() { return resiliencyOptions; } + @Override + public String toString() { + return String.format( + "{channelPoolOptions: %s, metricsOptions: %s, resiliencyOptions: %s}", + getChannelPoolOptions(), + getMetricsOptions(), + getResiliencyOptions() + ); + } + /** Creates a new GcpManagedChannelOptions.Builder. */ public static Builder newBuilder() { return new Builder(); @@ -186,6 +197,16 @@ public boolean isUseRoundRobinOnBind() { return useRoundRobinOnBind; } + @Override + public String toString() { + return String.format( + "{maxSize: %d, concurrentStreamsLowWatermark: %d, useRoundRobinOnBind: %s}", + getMaxSize(), + getConcurrentStreamsLowWatermark(), + isUseRoundRobinOnBind() + ); + } + public static class Builder { private int maxSize = GcpManagedChannel.DEFAULT_MAX_CHANNEL; private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM; @@ -273,6 +294,27 @@ public String getNamePrefix() { return namePrefix; } + @Override + public String toString() { + Iterator keyIterator = getLabelKeys().iterator(); + Iterator valueIterator = getLabelValues().iterator(); + + final List labels = new ArrayList<>(); + while (keyIterator.hasNext() && valueIterator.hasNext()) { + labels.add( + String.format( + "%s: \"%s\"", keyIterator.next().getKey(), valueIterator.next().getValue() + ) + ); + } + return String.format( + "{namePrefix: \"%s\", labels: [%s], metricRegistry: %s}", + getNamePrefix(), + String.join(", ", labels), + getMetricRegistry() + ); + } + /** Creates a new GcpMetricsOptions.Builder. */ public static Builder newBuilder() { return new Builder(); @@ -385,6 +427,18 @@ public int getUnresponsiveDetectionDroppedCount() { return unresponsiveDetectionDroppedCount; } + @Override + public String toString() { + return String.format( + "{notReadyFallbackEnabled: %s, unresponsiveDetectionEnabled: %s, " + + "unresponsiveDetectionMs: %d, unresponsiveDetectionDroppedCount: %d}", + isNotReadyFallbackEnabled(), + isUnresponsiveDetectionEnabled(), + getUnresponsiveDetectionMs(), + getUnresponsiveDetectionDroppedCount() + ); + } + public static class Builder { private boolean notReadyFallbackEnabled = false; private boolean unresponsiveDetectionEnabled = false; diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index e3b5ff72..0c4664c3 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -618,8 +618,9 @@ public void testMetrics() { // Logs metrics options. assertThat(logRecords.get(logRecords.size() - 2).getLevel()).isEqualTo(Level.FINE); - assertThat(logRecords.get(logRecords.size() - 2).getMessage()).isEqualTo( - poolIndex + ": Metrics name prefix = \"some/prefix/\", tags: key_a = val_a, key_b = val_b" + assertThat(logRecords.get(logRecords.size() - 2).getMessage()).startsWith( + poolIndex + ": Metrics options: {namePrefix: \"some/prefix/\", labels: " + + "[key_a: \"val_a\", key_b: \"val_b\"]," ); assertThat(lastLogLevel()).isEqualTo(Level.INFO); From 61b8af99a5240931146d301c8134863528abe9d5 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 1 Jun 2022 15:04:29 -0700 Subject: [PATCH 69/87] Log active streams, calls, and affinity counts for each channel. --- .../google/cloud/grpc/GcpManagedChannel.java | 59 +++- .../cloud/grpc/GcpManagedChannelTest.java | 291 ++++++++++++++++++ 2 files changed, 339 insertions(+), 11 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index d895b84a..76de0fc6 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -25,6 +25,7 @@ import com.google.cloud.grpc.proto.ApiConfig; import com.google.cloud.grpc.proto.MethodConfig; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.MessageOrBuilder; @@ -135,8 +136,8 @@ public class GcpManagedChannel extends ManagedChannel { private final AtomicLong totalErrCalls = new AtomicLong(); private boolean minErrReported = false; private boolean maxErrReported = false; - private int minAffinity = 0; - private int maxAffinity = 0; + private final AtomicInteger minAffinity = new AtomicInteger(); + private final AtomicInteger maxAffinity = new AtomicInteger(); private final AtomicInteger totalAffinityCount = new AtomicInteger(); private final AtomicLong fallbacksSucceeded = new AtomicLong(); private final AtomicLong fallbacksFailed = new AtomicLong(); @@ -239,6 +240,19 @@ private void logMetricsOptions() { } } + private void logChannelsStats() { + logger.fine(log( + "Active streams counts: [%s]", Joiner.on(", ").join( + channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).iterator() + ) + )); + logger.fine(log( + "Affinity counts: [%s]", Joiner.on(", ").join( + channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).iterator() + ) + )); + } + private void initMetrics() { final GcpMetricsOptions metricsOptions = options.getMetricsOptions(); if (metricsOptions == null) { @@ -464,8 +478,10 @@ private void logCumulative(String key, long value) { })); } - private void logMetrics() { + @VisibleForTesting + void logMetrics() { logMetricsOptions(); + logChannelsStats(); reportMinReadyChannels(); reportMaxReadyChannels(); reportMaxChannels(); @@ -654,15 +670,17 @@ private int reportMaxTotalActiveStreams() { } private int reportMinAffinity() { - int value = minAffinity; - minAffinity = channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).min().orElse(0); + int value = minAffinity.getAndSet( + channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).min().orElse(0) + ); logGauge(GcpMetricsConstants.METRIC_MIN_AFFINITY, value); return value; } private int reportMaxAffinity() { - int value = maxAffinity; - maxAffinity = channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).max().orElse(0); + int value = maxAffinity.getAndSet( + channelRefs.stream().mapToInt(ChannelRef::getAffinityCount).max().orElse(0) + ); logGauge(GcpMetricsConstants.METRIC_MAX_AFFINITY, value); return value; } @@ -693,6 +711,23 @@ private long reportTotalOkCalls() { return value; } + private LongSummaryStatistics calcStatsAndLog(String logLabel, ToLongFunction func) { + StringBuilder str = new StringBuilder(logLabel + ": ["); + final LongSummaryStatistics stats = + channelRefs.stream().mapToLong(ch -> { + long count = func.applyAsLong(ch); + if (str.charAt(str.length() - 1) != '[') { + str.append(", "); + } + str.append(count); + return count; + }).summaryStatistics(); + + str.append("]"); + logger.fine(log(str.toString())); + return stats; + } + private void calcMinMaxOkCalls() { if (minOkReported && maxOkReported) { minOkReported = false; @@ -700,7 +735,7 @@ private void calcMinMaxOkCalls() { return; } final LongSummaryStatistics stats = - channelRefs.stream().mapToLong(ChannelRef::getAndResetOkCalls).summaryStatistics(); + calcStatsAndLog("Ok calls", ChannelRef::getAndResetOkCalls); minOkCalls = stats.getMin(); maxOkCalls = stats.getMax(); } @@ -732,7 +767,7 @@ private void calcMinMaxErrCalls() { return; } final LongSummaryStatistics stats = - channelRefs.stream().mapToLong(ChannelRef::getAndResetErrCalls).summaryStatistics(); + calcStatsAndLog("Failed calls", ChannelRef::getAndResetErrCalls); minErrCalls = stats.getMin(); maxErrCalls = stats.getMax(); } @@ -1374,12 +1409,14 @@ protected int getId() { } protected void affinityCountIncr() { - affinityCount.incrementAndGet(); + int count = affinityCount.incrementAndGet(); + maxAffinity.getAndUpdate(currentMax -> Math.max(currentMax, count)); totalAffinityCount.incrementAndGet(); } protected void affinityCountDecr() { - affinityCount.decrementAndGet(); + int count = affinityCount.decrementAndGet(); + minAffinity.getAndUpdate(currentMin -> Math.min(currentMin, count)); totalAffinityCount.decrementAndGet(); } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 0c4664c3..1c0f7dfe 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -52,6 +52,8 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Handler; @@ -709,6 +711,221 @@ public void testMetrics() { } } + @Test + public void testLogMetrics() throws InterruptedException { + // Watch debug messages. + testLogger.setLevel(Level.FINE); + + final GcpManagedChannel pool = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withOptions( + GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions( + GcpChannelPoolOptions.newBuilder() + .setMaxSize(5) + .setConcurrentStreamsLowWatermark(3) + .build()) + .withMetricsOptions( + GcpMetricsOptions.newBuilder() + .withNamePrefix("prefix") + .build()) + .withResiliencyOptions( + GcpResiliencyOptions.newBuilder() + .setNotReadyFallback(true) + .withUnresponsiveConnectionDetection(100, 2) + .build()) + .build()) + .build(); + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try { + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String poolIndex = String.format("pool-%d", currentIndex); + + int[] streams = new int[]{3, 2, 5, 7, 1}; + int[] keyCount = new int[]{2, 3, 1, 1, 4}; + int[] okCalls = new int[]{2, 2, 8, 2, 3}; + int[] errCalls = new int[]{1, 1, 2, 2, 1}; + List channels = new ArrayList<>(); + for (int i = 0; i < streams.length; i++) { + FakeManagedChannel channel = new FakeManagedChannel(executorService); + channels.add(channel); + ChannelRef ref = pool.new ChannelRef(channel, i); + pool.channelRefs.add(ref); + + // Simulate channel connecting. + channel.setState(ConnectivityState.CONNECTING); + TimeUnit.MILLISECONDS.sleep(10); + + // For the last one... + if (i == streams.length - 1) { + // This will be a couple of successful fallbacks. + pool.getChannelRef(null); + pool.getChannelRef(null); + // Bring down all other channels. + for (int j = 0; j < i; j++) { + channels.get(j).setState(ConnectivityState.CONNECTING); + } + TimeUnit.MILLISECONDS.sleep(100); + // And this will be a failed fallback (no ready channels). + pool.getChannelRef(null); + + // Simulate unresponsive connection. + long startNanos = System.nanoTime(); + final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED); + ref.activeStreamsCountIncr(); + ref.activeStreamsCountDecr(startNanos, deStatus, false); + ref.activeStreamsCountIncr(); + ref.activeStreamsCountDecr(startNanos, deStatus, false); + + // Simulate unresponsive connection with more dropped calls. + startNanos = System.nanoTime(); + ref.activeStreamsCountIncr(); + ref.activeStreamsCountDecr(startNanos, deStatus, false); + ref.activeStreamsCountIncr(); + ref.activeStreamsCountDecr(startNanos, deStatus, false); + TimeUnit.MILLISECONDS.sleep(110); + ref.activeStreamsCountIncr(); + ref.activeStreamsCountDecr(startNanos, deStatus, false); + } + + channel.setState(ConnectivityState.READY); + + for (int j = 0; j < streams[i]; j++) { + ref.activeStreamsCountIncr(); + } + // Bind affinity keys. + final List keys = new ArrayList<>(); + for (int j = 0; j < keyCount[i]; j++) { + keys.add("key-" + i + "-" + j); + } + pool.bind(ref, keys); + // Simulate successful calls. + for (int j = 0; j < okCalls[i]; j++) { + ref.activeStreamsCountDecr(0, Status.OK, false); + ref.activeStreamsCountIncr(); + } + // Simulate failed calls. + for (int j = 0; j < errCalls[i]; j++) { + ref.activeStreamsCountDecr(0, Status.UNAVAILABLE, false); + ref.activeStreamsCountIncr(); + } + + } + + logRecords.clear(); + + pool.logMetrics(); + + List messages = Arrays.asList(logRecords.stream().map(LogRecord::getMessage).toArray()); + + assertThat(messages).contains(poolIndex + ": Active streams counts: [3, 2, 5, 7, 1]"); + assertThat(messages).contains(poolIndex + ": Affinity counts: [2, 3, 1, 1, 4]"); + + assertThat(messages).contains(poolIndex + ": stat: min_ready_channels = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_ready_channels = 4"); + assertThat(messages).contains(poolIndex + ": stat: max_channels = 5"); + assertThat(messages).contains(poolIndex + ": stat: max_allowed_channels = 5"); + assertThat(messages).contains(poolIndex + ": stat: num_channel_disconnect = 4"); + assertThat(messages).contains(poolIndex + ": stat: num_channel_connect = 5"); + assertThat(messages.stream().filter(o -> o.toString().matches( + poolIndex + ": stat: min_channel_readiness_time = \\d\\d+" + ) + ).count()).isEqualTo(1); + assertThat(messages.stream().filter(o -> o.toString().matches( + poolIndex + ": stat: avg_channel_readiness_time = \\d\\d+" + ) + ).count()).isEqualTo(1); + assertThat(messages.stream().filter(o -> o.toString().matches( + poolIndex + ": stat: max_channel_readiness_time = \\d\\d+" + ) + ).count()).isEqualTo(1); + assertThat(messages).contains(poolIndex + ": stat: min_active_streams_per_channel = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_active_streams_per_channel = 7"); + assertThat(messages).contains(poolIndex + ": stat: min_total_active_streams = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_total_active_streams = 18"); + assertThat(messages).contains(poolIndex + ": stat: min_affinity_per_channel = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_affinity_per_channel = 4"); + assertThat(messages).contains(poolIndex + ": stat: num_affinity = 11"); + assertThat(messages).contains(poolIndex + ": Ok calls: [2, 2, 8, 2, 3]"); + assertThat(messages).contains(poolIndex + ": Failed calls: [1, 1, 2, 2, 6]"); + assertThat(messages).contains(poolIndex + ": stat: min_calls_per_channel_ok = 2"); + assertThat(messages).contains(poolIndex + ": stat: min_calls_per_channel_err = 1"); + assertThat(messages).contains(poolIndex + ": stat: max_calls_per_channel_ok = 8"); + assertThat(messages).contains(poolIndex + ": stat: max_calls_per_channel_err = 6"); + assertThat(messages).contains(poolIndex + ": stat: num_calls_completed_ok = 17"); + assertThat(messages).contains(poolIndex + ": stat: num_calls_completed_err = 12"); + assertThat(messages).contains(poolIndex + ": stat: num_fallbacks_ok = 2"); + assertThat(messages).contains(poolIndex + ": stat: num_fallbacks_fail = 1"); + assertThat(messages).contains(poolIndex + ": stat: num_unresponsive_detections = 2"); + assertThat(messages.stream().filter(o -> o.toString().matches( + poolIndex + ": stat: min_unresponsive_detection_time = 1\\d\\d" + ) + ).count()).isEqualTo(1); + assertThat(messages.stream().filter(o -> o.toString().matches( + poolIndex + ": stat: max_unresponsive_detection_time = 1\\d\\d" + ) + ).count()).isEqualTo(1); + assertThat(messages).contains(poolIndex + ": stat: min_unresponsive_dropped_calls = 2"); + assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 3"); + + assertThat(logRecords.size()).isEqualTo(34); + logRecords.forEach(logRecord -> + assertThat(logRecord.getLevel()).named(logRecord.getMessage()).isEqualTo(Level.FINE) + ); + + logRecords.clear(); + + // Next call should update minimums that was 0 previously (e.g., min_ready_channels, + // min_active_streams_per_channel, min_total_active_streams...). + pool.logMetrics(); + + messages = Arrays.asList(logRecords.stream().map(LogRecord::getMessage).toArray()); + + assertThat(messages).contains(poolIndex + ": Active streams counts: [3, 2, 5, 7, 1]"); + assertThat(messages).contains(poolIndex + ": Affinity counts: [2, 3, 1, 1, 4]"); + + assertThat(messages).contains(poolIndex + ": stat: min_ready_channels = 1"); + assertThat(messages).contains(poolIndex + ": stat: max_ready_channels = 1"); + assertThat(messages).contains(poolIndex + ": stat: max_channels = 5"); + assertThat(messages).contains(poolIndex + ": stat: max_allowed_channels = 5"); + assertThat(messages).contains(poolIndex + ": stat: num_channel_disconnect = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_channel_connect = 0"); + assertThat(messages).contains(poolIndex + ": stat: min_channel_readiness_time = 0"); + assertThat(messages).contains(poolIndex + ": stat: avg_channel_readiness_time = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_channel_readiness_time = 0"); + assertThat(messages).contains(poolIndex + ": stat: min_active_streams_per_channel = 1"); + assertThat(messages).contains(poolIndex + ": stat: max_active_streams_per_channel = 7"); + assertThat(messages).contains(poolIndex + ": stat: min_total_active_streams = 18"); + assertThat(messages).contains(poolIndex + ": stat: max_total_active_streams = 18"); + assertThat(messages).contains(poolIndex + ": stat: min_affinity_per_channel = 1"); + assertThat(messages).contains(poolIndex + ": stat: max_affinity_per_channel = 4"); + assertThat(messages).contains(poolIndex + ": stat: num_affinity = 11"); + assertThat(messages).contains(poolIndex + ": Ok calls: [0, 0, 0, 0, 0]"); + assertThat(messages).contains(poolIndex + ": Failed calls: [0, 0, 0, 0, 0]"); + assertThat(messages).contains(poolIndex + ": stat: min_calls_per_channel_ok = 0"); + assertThat(messages).contains(poolIndex + ": stat: min_calls_per_channel_err = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_calls_per_channel_ok = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_calls_per_channel_err = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_calls_completed_ok = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_calls_completed_err = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_fallbacks_ok = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_fallbacks_fail = 0"); + assertThat(messages).contains(poolIndex + ": stat: num_unresponsive_detections = 0"); + assertThat(messages).contains(poolIndex + ": stat: min_unresponsive_detection_time = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_detection_time = 0"); + assertThat(messages).contains(poolIndex + ": stat: min_unresponsive_dropped_calls = 0"); + assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 0"); + + assertThat(logRecords.size()).isEqualTo(34); + + } finally { + pool.shutdownNow(); + executorService.shutdownNow(); + } + } + @Test public void testUnresponsiveDetection() throws InterruptedException { // Watch debug messages. @@ -858,6 +1075,80 @@ public void testUnresponsiveDetection() throws InterruptedException { poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 0"); } + static class FakeManagedChannel extends ManagedChannel { + private ConnectivityState state = ConnectivityState.IDLE; + private Runnable stateCallback; + private final ExecutorService exec; + + FakeManagedChannel(ExecutorService exec) { + this.exec = exec; + } + + @Override + public void enterIdle() {} + + @Override + public ConnectivityState getState(boolean requestConnection) { + return state; + } + + public void setState(ConnectivityState state) { + if (state.equals(this.state)) { + return; + } + this.state = state; + if (stateCallback != null) { + exec.execute(stateCallback); + stateCallback = null; + } + } + + @Override + public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) { + if (!source.equals(state)) { + exec.execute(callback); + return; + } + stateCallback = callback; + } + + @Override + public ManagedChannel shutdown() { + return null; + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public ManagedChannel shutdownNow() { + return null; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + + @Override + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + return null; + } + + @Override + public String authority() { + return null; + } + } + static class FakeIdleCountingManagedChannel extends ManagedChannel { private final AtomicInteger idleCounter; From 503ebcb868afb4110c29475b6c96c13fcae8637e Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 1 Jun 2022 15:55:38 -0700 Subject: [PATCH 70/87] Log channel creation, state change, and forcing to idle state. --- .../google/cloud/grpc/GcpManagedChannel.java | 20 +++++++++++--- .../cloud/grpc/GcpManagedChannelTest.java | 26 +++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 76de0fc6..52e60298 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -886,6 +886,9 @@ public void run() { return; } ConnectivityState newState = channel.getState(false); + logger.finer( + log("Channel %d state change detected: %s -> %s", channelId, currentState, newState) + ); if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) { incReadyChannels(); saveReadinessTime(System.nanoTime() - connectingStartNanos); @@ -1030,6 +1033,7 @@ private synchronized ChannelRef createNewChannel() { final int size = channelRefs.size(); ChannelRef channelRef = new ChannelRef(delegateChannelBuilder.build(), size); channelRefs.add(channelRef); + logger.finer(log("Channel %d created.", channelRef.getId())); return channelRef; } @@ -1481,7 +1485,7 @@ private void detectUnresponsiveConnection( return; } if (deadlineExceededCount.incrementAndGet() >= unresponsiveDropCount - && unresponsiveTimingConditionMet()) { + && msSinceLastResponse() >= unresponsiveMs) { maybeReconnectUnresponsive(); } return; @@ -1493,15 +1497,23 @@ && unresponsiveTimingConditionMet()) { } } - private boolean unresponsiveTimingConditionMet() { - return (System.nanoTime() - lastResponseNanos) / 1000000 >= unresponsiveMs; + private long msSinceLastResponse() { + return (System.nanoTime() - lastResponseNanos) / 1000000; } private synchronized void maybeReconnectUnresponsive() { + final long msSinceLastResponse = msSinceLastResponse(); if (deadlineExceededCount.get() >= unresponsiveDropCount - && unresponsiveTimingConditionMet()) { + && msSinceLastResponse >= unresponsiveMs) { recordUnresponsiveDetection( System.nanoTime() - lastResponseNanos, deadlineExceededCount.get()); + logger.finer(log( + "Channel %d connection is unresponsive for %d ms and %d deadline exceeded calls. " + + "Forcing channel to idle state.", + channelId, + msSinceLastResponse, + deadlineExceededCount.get() + )); delegate.enterIdle(); lastResponseNanos = System.nanoTime(); deadlineExceededCount.set(0); diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 1c0f7dfe..eb6aa1c1 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -213,10 +213,27 @@ public void testPoolOptionsOverrideApiConfig() { @Test public void testGetChannelRefInitialization() { + // Watch debug messages. + testLogger.setLevel(Level.FINER); + + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String poolIndex = String.format("pool-%d", currentIndex); + + // Initial log messages count. + int logCount = logRecords.size(); + // Should not have a managedchannel by default. assertEquals(0, gcpChannel.channelRefs.size()); // But once requested it's there. assertEquals(0, gcpChannel.getChannelRef(null).getAffinityCount()); + + assertThat(logRecords.size()).isEqualTo(logCount + 2); + assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Channel 0 created."); + assertThat(lastLogLevel()).isEqualTo(Level.FINER); + assertThat(logRecords.get(logRecords.size() - 2).getMessage()).isEqualTo( + poolIndex + ": Channel 0 state change detected: null -> IDLE"); + assertThat(logRecords.get(logRecords.size() - 2).getLevel()).isEqualTo(Level.FINER); + // The state of this channel is idle. assertEquals(ConnectivityState.IDLE, gcpChannel.getState(false)); assertEquals(1, gcpChannel.channelRefs.size()); @@ -929,7 +946,7 @@ public void testLogMetrics() throws InterruptedException { @Test public void testUnresponsiveDetection() throws InterruptedException { // Watch debug messages. - testLogger.setLevel(Level.FINE); + testLogger.setLevel(Level.FINER); final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry(); // Creating a pool with unresponsive connection detection for 100 ms, 3 dropped requests. final GcpManagedChannel pool = @@ -1053,17 +1070,22 @@ public void testUnresponsiveDetection() throws InterruptedException { // Any subsequent deadline exceeded after 100ms must trigger the reconnection. chRef.activeStreamsCountDecr(startNanos, deStatus, false); assertEquals(2, idleCounter.get()); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).matches( + poolIndex + ": Channel 0 connection is unresponsive for 1\\d\\d ms and 4 deadline " + + "exceeded calls. Forcing channel to idle state."); + assertThat(lastLogLevel()).isEqualTo(Level.FINER); // The cumulative num_unresponsive_detections metric must become 2. metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); assertThat(metric.size()).isEqualTo(1); assertThat(metric.get(0).value()).isEqualTo(2L); assertThat(logRecords.size()).isEqualTo(++logCount); - assertThat(lastLogLevel()).isEqualTo(Level.FINE); // But the log metric count the detections since previous report for num_unresponsive_detections // in the logs. It is always delta in the logs, not cumulative. assertThat(lastLogMessage()).isEqualTo( poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 1"); + assertThat(lastLogLevel()).isEqualTo(Level.FINE); // If we log it again the cumulative metric value must remain unchanged. metric = record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS); assertThat(metric.size()).isEqualTo(1); From bb0b0a5eab8ed97b4db45a0076d63c1661dd4e71 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 1 Jun 2022 16:11:33 -0700 Subject: [PATCH 71/87] Log bind/unbind affinity keys. --- .../google/cloud/grpc/GcpManagedChannel.java | 9 +++++++ .../cloud/grpc/GcpManagedChannelTest.java | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 52e60298..fc3e7b98 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -1249,6 +1249,12 @@ protected void bind(ChannelRef channelRef, List affinityKeys) { if (channelRef == null || affinityKeys == null) { return; } + logger.finest(log( + "Binding %d key(s) to channel %d: [%s]", + affinityKeys.size(), + channelRef.getId(), + String.join(", ", affinityKeys) + )); for (String affinityKey : affinityKeys) { while (affinityKeyToChannelRef.putIfAbsent(affinityKey, channelRef) != null) { unbind(Collections.singletonList(affinityKey)); @@ -1266,6 +1272,9 @@ protected void unbind(List affinityKeys) { ChannelRef channelRef = affinityKeyToChannelRef.remove(affinityKey); if (channelRef != null) { channelRef.affinityCountDecr(); + logger.finest(log("Unbinding key %s from channel %d.", affinityKey, channelRef.getId())); + } else { + logger.finest(log("Unbinding key %s but it wasn't bound.", affinityKey)); } } } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index eb6aa1c1..86b408e7 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -432,13 +432,30 @@ public void testGetChannelRefMaxSize() { @Test public void testBindUnbindKey() { + // Watch debug messages. + testLogger.setLevel(Level.FINEST); + + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String poolIndex = String.format("pool-%d", currentIndex); + // Initialize the channel and bind the key, check the affinity count. ChannelRef cf1 = gcpChannel.new ChannelRef(builder.build(), 1, 0, 5); ChannelRef cf2 = gcpChannel.new ChannelRef(builder.build(), 2, 0, 4); gcpChannel.channelRefs.add(cf1); gcpChannel.channelRefs.add(cf2); + gcpChannel.bind(cf1, Collections.singletonList("key1")); + + // Initial log messages count. + int logCount = logRecords.size(); + gcpChannel.bind(cf2, Collections.singletonList("key2")); + + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Binding 1 key(s) to channel 2: [key2]"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + gcpChannel.bind(cf2, Collections.singletonList("key3")); // Binding the same key to the same channel should not increase affinity count. gcpChannel.bind(cf1, Collections.singletonList("key1")); @@ -451,15 +468,25 @@ public void testBindUnbindKey() { assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount()); assertEquals(3, gcpChannel.affinityKeyToChannelRef.size()); + logCount = logRecords.size(); + // Unbind the affinity key. gcpChannel.unbind(Collections.singletonList("key1")); assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount()); assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount()); assertEquals(2, gcpChannel.affinityKeyToChannelRef.size()); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Unbinding key key1 from channel 1."); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); gcpChannel.unbind(Collections.singletonList("key1")); assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount()); assertEquals(1, gcpChannel.channelRefs.get(1).getAffinityCount()); assertEquals(2, gcpChannel.affinityKeyToChannelRef.size()); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Unbinding key key1 but it wasn't bound."); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); gcpChannel.unbind(Collections.singletonList("key2")); assertEquals(1, gcpChannel.channelRefs.get(0).getAffinityCount()); assertEquals(0, gcpChannel.channelRefs.get(1).getAffinityCount()); From 61404d872630493a8711590bbc65df613e8b4392 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 8 Jun 2022 12:39:47 -0700 Subject: [PATCH 72/87] Log channel chosen for a bind call. --- .../google/cloud/grpc/GcpManagedChannel.java | 8 ++- .../cloud/grpc/SpannerIntegrationTest.java | 58 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index fc3e7b98..8c8f729f 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -949,10 +949,14 @@ public int getMaxActiveStreams() { * @return {@link ChannelRef} channel to use for a call. */ protected ChannelRef getChannelRefForBind() { + ChannelRef channelRef; if (options.getChannelPoolOptions() != null && options.getChannelPoolOptions().isUseRoundRobinOnBind()) { - return getChannelRefRoundRobin(); + channelRef = getChannelRefRoundRobin(); + } else { + channelRef = getChannelRef(null); } - return getChannelRef(null); + logger.finest(log("Channel %d picked for bind operation.", channelRef.getId())); + return channelRef; } /** diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java index 8939ae13..0d3d3414 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java @@ -73,12 +73,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -94,6 +99,8 @@ @RunWith(JUnit4.class) public final class SpannerIntegrationTest { + private static final Logger testLogger = Logger.getLogger(GcpManagedChannel.class.getName()); + private final List logRecords = new LinkedList<>(); private static final String GCP_PROJECT_ID = System.getenv("GCP_PROJECT_ID"); private static final String INSTANCE_ID = "grpc-gcp-test-instance"; private static final String DB_NAME = "grpc-gcp-test-db"; @@ -128,6 +135,31 @@ public static void beforeClass() { initializeTable(databaseClient); } + private String lastLogMessage() { + return lastLogMessage(1); + } + + private String lastLogMessage(int nthFromLast) { + return logRecords.get(logRecords.size() - nthFromLast).getMessage(); + } + + private Level lastLogLevel() { + return logRecords.get(logRecords.size() - 1).getLevel(); + } + + private final Handler testLogHandler = new Handler() { + @Override + public void publish(LogRecord record) { + logRecords.add(record); + } + + @Override + public void flush() {} + + @Override + public void close() throws SecurityException {} + }; + private static void initializeTable(DatabaseClient databaseClient) { List mutations = Arrays.asList( @@ -370,6 +402,7 @@ private void deleteFutureSessions(SpannerFutureStub stub, List futureNam @Before public void setupChannels() { + testLogger.addHandler(testLogHandler); File configFile = new File(SpannerIntegrationTest.class.getClassLoader().getResource(API_FILE).getFile()); gcpChannel = @@ -394,6 +427,8 @@ public void setupChannels() { @After public void shutdownChannels() { + testLogger.removeHandler(testLogHandler); + testLogger.setLevel(Level.INFO); gcpChannel.shutdownNow(); gcpChannelBRR.shutdownNow(); } @@ -498,15 +533,32 @@ public void testSessionsCreatedUsingRoundRobin() throws Exception { @Test public void testSessionsCreatedWithoutRoundRobin() throws Exception { + // Watch debug messages. + testLogger.setLevel(Level.FINEST); + final int currentIndex = GcpManagedChannel.channelPoolIndex.get() - 1; + final String poolIndex = String.format("pool-%d", currentIndex); + SpannerFutureStub stub = getSpannerFutureStub(); List> futures = new ArrayList<>(); assertEquals(ConnectivityState.IDLE, gcpChannel.getState(false)); + // Initial log messages count. + int logCount = logRecords.size(); + // Should create one session per channel. CreateSessionRequest req = CreateSessionRequest.newBuilder().setDatabase(DATABASE_PATH).build(); for (int i = 0; i < MAX_CHANNEL; i++) { ListenableFuture future = stub.createSession(req); futures.add(future); + assertThat(lastLogMessage(3)).isEqualTo( + poolIndex + ": Channel " + i + " state change detected: null -> IDLE"); + assertThat(lastLogMessage(2)).isEqualTo( + poolIndex + ": Channel " + i + " created."); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Channel " + i + " picked for bind operation."); + logCount += 3; + assertThat(logRecords.size()).isEqualTo(logCount); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); } // Each channel should have 1 active stream with the CreateSession request because we create them concurrently. checkChannelRefs(gcpChannel, MAX_CHANNEL, 1, 0); @@ -534,11 +586,17 @@ public void testSessionsCreatedWithoutRoundRobin() throws Exception { // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); + logCount = logRecords.size(); + // Create another 1 session per channel sequentially. // Without the round-robin it won't use the currentChannel as it has more active streams (1) than other channels. for (int i = 0; i < MAX_CHANNEL; i++) { ListenableFuture future = stub.createSession(req); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Channel 0 picked for bind operation."); + assertThat(logRecords.size()).isEqualTo(++logCount); future.get(); + logCount++; // For session mapping log message. } ResultSet response = responseFuture.get(); From c1b4f8a83fd273f92ae418636623a0351e4e17c3 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 8 Jun 2022 17:11:45 -0700 Subject: [PATCH 73/87] Log fallback successes and failures. --- .../google/cloud/grpc/GcpManagedChannel.java | 14 ++- .../cloud/grpc/GcpManagedChannelTest.java | 87 ++++++++++++++++--- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 8c8f729f..fd963a82 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -1009,6 +1009,7 @@ protected ChannelRef getChannelRef(@Nullable String key) { Integer channelId = tempMap.get(key); if (channelId != null && !fallbackMap.containsKey(channelId)) { // Fallback channel is ready. + logger.finest(log("Using fallback channel: %d -> %d", mappedChannel.getId(), channelId)); fallbacksSucceeded.incrementAndGet(); return channelRefs.get(channelId); } @@ -1018,11 +1019,15 @@ protected ChannelRef getChannelRef(@Nullable String key) { && channelRef.getActiveStreamsCount() < DEFAULT_MAX_STREAM) { // Got a ready and not an overloaded channel. if (channelRef.getId() != mappedChannel.getId()) { + logger.finest(log( + "Setting fallback channel: %d -> %d", mappedChannel.getId(), channelRef.getId() + )); fallbacksSucceeded.incrementAndGet(); tempMap.put(key, channelRef.getId()); } return channelRef; } + logger.finest(log("Failed to find fallback for channel %d", mappedChannel.getId())); fallbacksFailed.incrementAndGet(); if (channelId != null) { // Stick with previous mapping if fallback has failed. @@ -1081,7 +1086,8 @@ private ChannelRef pickLeastBusyChannel(boolean forFallback) { } if (channelRefs.size() < maxSize && readyMinStreams >= maxConcurrentStreamsLowWatermark) { - if (!forFallback) { + if (!forFallback && readyCandidate == null) { + logger.finest(log("Fallback to newly created channel")); fallbacksSucceeded.incrementAndGet(); } return createNewChannel(); @@ -1089,11 +1095,17 @@ private ChannelRef pickLeastBusyChannel(boolean forFallback) { if (readyCandidate != null) { if (!forFallback && readyCandidate.getId() != channelCandidate.getId()) { + logger.finest(log( + "Picking fallback channel: %d -> %d", channelCandidate.getId(), readyCandidate.getId())); fallbacksSucceeded.incrementAndGet(); } return readyCandidate; } + if (!forFallback) { + logger.finest(log("Failed to find fallback for channel %d", channelCandidate.getId())); + fallbacksFailed.incrementAndGet(); + } return channelCandidate; } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 86b408e7..7577a1c5 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -83,11 +83,19 @@ public final class GcpManagedChannelTest { private final List logRecords = new LinkedList<>(); private String lastLogMessage() { - return logRecords.get(logRecords.size() - 1).getMessage(); + return lastLogMessage(1); + } + + private String lastLogMessage(int nthFromLast) { + return logRecords.get(logRecords.size() - nthFromLast).getMessage(); } private Level lastLogLevel() { - return logRecords.get(logRecords.size() - 1).getLevel(); + return lastLogLevel(1); + } + + private Level lastLogLevel(int nthFromLast) { + return logRecords.get(logRecords.size() - nthFromLast).getLevel(); } private final Handler testLogHandler = new Handler() { @@ -277,6 +285,9 @@ private void assertFallbacksMetric( @Test public void testGetChannelRefWithFallback() { + // Watch debug messages. + testLogger.setLevel(Level.FINEST); + final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry(); final int maxSize = 3; @@ -303,6 +314,9 @@ public void testGetChannelRefWithFallback() { .build()) .build(); + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String poolIndex = String.format("pool-%d", currentIndex); + // Creates the first channel with 0 id. assertEquals(0, pool.getNumberOfChannels()); ChannelRef chRef = pool.getChannelRef(null); @@ -317,16 +331,22 @@ public void testGetChannelRefWithFallback() { // Let's simulate the non-ready state for the 0 channel. pool.processChannelStateChange(0, ConnectivityState.CONNECTING); + int logCount = logRecords.size(); // Now request for a channel should return a newly created channel because our current channel - // is not ready and we haven't reached the pool's max size. + // is not ready, and we haven't reached the pool's max size. chRef = pool.getChannelRef(null); assertEquals(1, chRef.getId()); assertEquals(2, pool.getNumberOfChannels()); // This was a fallback from non-ready channel 0 to the newly created channel 1. + assertThat(logRecords.size()).isEqualTo(logCount + 3); + assertThat(lastLogMessage(3)).isEqualTo( + poolIndex + ": Fallback to newly created channel"); + assertThat(lastLogLevel(3)).isEqualTo(Level.FINEST); assertFallbacksMetric(fakeRegistry, 1, 0); // Adding one active stream to channel 1. pool.channelRefs.get(1).activeStreamsCountIncr(); + logCount = logRecords.size(); // Having 0 active streams on channel 0 and 1 active streams on channel one with the default // settings would return channel 0 for the next channel request. But having fallback enabled and // channel 0 not ready it should return channel 1 instead. @@ -334,6 +354,10 @@ public void testGetChannelRefWithFallback() { assertEquals(1, chRef.getId()); assertEquals(2, pool.getNumberOfChannels()); // This was the second fallback from non-ready channel 0 to the channel 1. + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Picking fallback channel: 0 -> 1"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); assertFallbacksMetric(fakeRegistry, 2, 0); // Now let's have channel 0 still as not ready but bring channel 1 streams to low watermark. @@ -345,8 +369,6 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(2, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); - // This was the third fallback from non-ready channel 0 to the newly created channel 2. - assertFallbacksMetric(fakeRegistry, 3, 0); // Now we reached max pool size. Let's bring channel 2 to the low watermark and channel 1 to the // low watermark + 1 streams. @@ -363,8 +385,8 @@ public void testGetChannelRefWithFallback() { chRef = pool.getChannelRef(null); assertEquals(2, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); - // This was the fourth fallback from non-ready channel 0 to the channel 2. - assertFallbacksMetric(fakeRegistry, 4, 0); + // This was the third fallback from non-ready channel 0 to the channel 2. + assertFallbacksMetric(fakeRegistry, 3, 0); // Let's bring channel 1 to max streams and mark channel 2 as not ready. for (int i = 0; i < MAX_STREAM - lowWatermark; i++) { @@ -375,47 +397,88 @@ public void testGetChannelRefWithFallback() { // Now we have two non-ready channels and one overloaded. // Even when fallback enabled there is no good candidate at this time, the next channel request - // should return a channel with lowest streams count regardless of its readiness state. + // should return a channel with the lowest streams count regardless of its readiness state. // In our case it is channel 0. + logCount = logRecords.size(); chRef = pool.getChannelRef(null); assertEquals(0, chRef.getId()); assertEquals(3, pool.getNumberOfChannels()); + // This will also count as a failed fallback because we couldn't find a ready and non-overloaded + // channel. + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Failed to find fallback for channel 0"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + assertFallbacksMetric(fakeRegistry, 3, 1); // Let's have an affinity key and bind it to channel 0. final String key = "ABC"; pool.bind(pool.channelRefs.get(0), Collections.singletonList(key)); + logCount = logRecords.size(); // Channel 0 is not ready currently and the fallback enabled should look for a fallback but we // still don't have a good channel because channel 1 is not ready and channel 2 is overloaded. // The getChannelRef should return the original channel 0 and report a failed fallback. chRef = pool.getChannelRef(key); assertEquals(0, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 4, 1); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Failed to find fallback for channel 0"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + assertFallbacksMetric(fakeRegistry, 3, 2); // Let's return channel 1 to a ready state. pool.processChannelStateChange(1, ConnectivityState.READY); + logCount = logRecords.size(); // Now we have a fallback candidate. // The getChannelRef should return the channel 1 and report a successful fallback. chRef = pool.getChannelRef(key); assertEquals(1, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 5, 1); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Setting fallback channel: 0 -> 1"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + assertFallbacksMetric(fakeRegistry, 4, 2); + + // Let's briefly bring channel 2 to ready state. + pool.processChannelStateChange(2, ConnectivityState.READY); + logCount = logRecords.size(); + // Now we have a better fallback candidate (fewer streams on channel 2). But this time we + // already used channel 1 as a fallback, and we should stick to it instead of returning the + // original channel. + // The getChannelRef should return the channel 1 and report a successful fallback. + chRef = pool.getChannelRef(key); + assertEquals(1, chRef.getId()); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Using fallback channel: 0 -> 1"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + assertFallbacksMetric(fakeRegistry, 5, 2); + pool.processChannelStateChange(2, ConnectivityState.CONNECTING); // Let's bring channel 1 back to connecting state. pool.processChannelStateChange(1, ConnectivityState.CONNECTING); + logCount = logRecords.size(); // Now we don't have a good fallback candidate again. But this time we already used channel 1 // as a fallback and we should stick to it instead of returning the original channel. // The getChannelRef should return the channel 1 and report a failed fallback. chRef = pool.getChannelRef(key); assertEquals(1, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 5, 2); + assertThat(logRecords.size()).isEqualTo(++logCount); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Failed to find fallback for channel 0"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); + assertFallbacksMetric(fakeRegistry, 5, 3); // Finally, we bring both channel 1 and channel 0 to the ready state and we should get the // original channel 0 for the key without any fallbacks happening. pool.processChannelStateChange(1, ConnectivityState.READY); pool.processChannelStateChange(0, ConnectivityState.READY); + logCount = logRecords.size(); chRef = pool.getChannelRef(key); assertEquals(0, chRef.getId()); - assertFallbacksMetric(fakeRegistry, 5, 2); + assertThat(logRecords.size()).isEqualTo(logCount); + assertFallbacksMetric(fakeRegistry, 5, 3); } @Test From bd5400e6fbc6704c8f3e47d01fd62c5db43454bd Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 16 Jun 2022 17:44:05 -0700 Subject: [PATCH 74/87] Add minSize option. Minimum channel pool size is the number of channels that will be created on channel pool creation and always try to keep connection to the server. --- .../google/cloud/grpc/GcpManagedChannel.java | 21 ++++++++-- .../cloud/grpc/GcpManagedChannelOptions.java | 23 +++++++++++ .../grpc/GcpManagedChannelOptionsTest.java | 25 +++++++++++- .../cloud/grpc/GcpManagedChannelTest.java | 40 ++++++++++++++++++- 4 files changed, 104 insertions(+), 5 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 799407b3..92d4e410 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -78,6 +78,7 @@ public class GcpManagedChannel extends ManagedChannel { private final int unresponsiveMs; private final int unresponsiveDropCount; private int maxSize = DEFAULT_MAX_CHANNEL; + private int minSize = 0; private int maxConcurrentStreamsLowWatermark = DEFAULT_MAX_STREAM; @VisibleForTesting final Map methodToAffinity = new HashMap<>(); @@ -174,6 +175,7 @@ public GcpManagedChannel( unresponsiveMs = 0; unresponsiveDropCount = 0; } + initMinChannels(); } /** @@ -207,11 +209,18 @@ private String log(String message) { return String.format("%s: %s", metricPoolIndex, message); } + private synchronized void initMinChannels() { + while (minSize - getNumberOfChannels() > 0) { + createNewChannel(); + } + } + private void initOptions() { GcpManagedChannelOptions.GcpChannelPoolOptions poolOptions = options.getChannelPoolOptions(); if (poolOptions != null) { - this.maxSize = poolOptions.getMaxSize(); - this.maxConcurrentStreamsLowWatermark = poolOptions.getConcurrentStreamsLowWatermark(); + maxSize = poolOptions.getMaxSize(); + minSize = poolOptions.getMinSize(); + maxConcurrentStreamsLowWatermark = poolOptions.getConcurrentStreamsLowWatermark(); } initMetrics(); } @@ -864,7 +873,9 @@ public void run() { if (channel == null) { return; } - ConnectivityState newState = channel.getState(false); + // Keep minSize channels always connected. + boolean requestConnection = channelId < minSize; + ConnectivityState newState = channel.getState(requestConnection); if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) { incReadyChannels(); saveReadinessTime(System.nanoTime() - connectingStartNanos); @@ -901,6 +912,10 @@ public int getMaxSize() { return maxSize; } + public int getMinSize() { + return minSize; + } + public int getNumberOfChannels() { return channelRefs.size(); } diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java index 9d5fe2d1..e58d0dce 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java @@ -152,6 +152,9 @@ public Builder withResiliencyOptions(GcpResiliencyOptions resiliencyOptions) { public static class GcpChannelPoolOptions { // The maximum number of channels in the pool. private final int maxSize; + // The minimum size of the channel pool. This number of channels will be created and these + // channels will try to always keep connection to the server. + private final int minSize; // If every channel in the pool has at least this amount of concurrent streams then a new channel will be created // in the pool unless the pool reached its maximum size. private final int concurrentStreamsLowWatermark; @@ -160,6 +163,7 @@ public static class GcpChannelPoolOptions { public GcpChannelPoolOptions(Builder builder) { maxSize = builder.maxSize; + minSize = builder.minSize; concurrentStreamsLowWatermark = builder.concurrentStreamsLowWatermark; useRoundRobinOnBind = builder.useRoundRobinOnBind; } @@ -168,6 +172,10 @@ public int getMaxSize() { return maxSize; } + public int getMinSize() { + return minSize; + } + public int getConcurrentStreamsLowWatermark() { return concurrentStreamsLowWatermark; } @@ -188,6 +196,7 @@ public boolean isUseRoundRobinOnBind() { public static class Builder { private int maxSize = GcpManagedChannel.DEFAULT_MAX_CHANNEL; + private int minSize = 0; private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM; private boolean useRoundRobinOnBind = false; @@ -199,6 +208,7 @@ public Builder(GcpChannelPoolOptions options) { return; } this.maxSize = options.getMaxSize(); + this.minSize = options.getMinSize(); this.concurrentStreamsLowWatermark = options.getConcurrentStreamsLowWatermark(); this.useRoundRobinOnBind = options.isUseRoundRobinOnBind(); } @@ -218,6 +228,19 @@ public Builder setMaxSize(int maxSize) { return this; } + /** + * Sets the minimum size of the channel pool. This number of channels will be created and + * these channels will try to always keep connection to the server established. + * + * @param minSize minimum number of channels the pool must have. + */ + public Builder setMinSize(int minSize) { + Preconditions.checkArgument(minSize >= 0, + "Channel pool minimum size must be 0 or positive."); + this.minSize = minSize; + return this; + } + /** * Sets the concurrent streams low watermark. * If every channel in the pool has at least this amount of concurrent streams then a new diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java index 3a39d86a..c8ddb9bd 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java @@ -16,11 +16,13 @@ package com.google.cloud.grpc; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions; import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions; import io.opencensus.metrics.LabelKey; @@ -34,7 +36,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Unit tests for GcpManagedChannel. */ +/** Unit tests for GcpManagedChannelOptionsTest. */ @RunWith(JUnit4.class) public final class GcpManagedChannelOptionsTest { private static final String namePrefix = "name-prefix"; @@ -168,4 +170,25 @@ public void testOptionsReBuild() { assertEquals(unresponsiveMs, resOpts.getUnresponsiveDetectionMs()); assertEquals(unresponsiveDroppedCount, resOpts.getUnresponsiveDetectionDroppedCount()); } + + @Test + public void testPoolOptions() { + final GcpManagedChannelOptions opts = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions( + GcpChannelPoolOptions.newBuilder() + .setMaxSize(5) + .setMinSize(2) + .setConcurrentStreamsLowWatermark(10) + .setUseRoundRobinOnBind(true) + .build() + ) + .build(); + + GcpChannelPoolOptions channelPoolOptions = opts.getChannelPoolOptions(); + assertThat(channelPoolOptions).isNotNull(); + assertThat(channelPoolOptions.getMaxSize()).isEqualTo(5); + assertThat(channelPoolOptions.getMinSize()).isEqualTo(2); + assertThat(channelPoolOptions.getConcurrentStreamsLowWatermark()).isEqualTo(10); + assertThat(channelPoolOptions.isUseRoundRobinOnBind()).isTrue(); + } } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index e3b5ff72..9b1308ed 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -169,6 +169,7 @@ public void testUsesPoolOptions() { resetGcpChannel(); GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() .setMaxSize(5) + .setMinSize(2) .setConcurrentStreamsLowWatermark(50) .build(); GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() @@ -179,8 +180,9 @@ public void testUsesPoolOptions() { GcpManagedChannelBuilder.forDelegateBuilder(builder) .withOptions(options) .build(); - assertEquals(0, gcpChannel.channelRefs.size()); + assertEquals(2, gcpChannel.channelRefs.size()); assertEquals(5, gcpChannel.getMaxSize()); + assertEquals(2, gcpChannel.getMinSize()); assertEquals(50, gcpChannel.getStreamsLowWatermark()); } @@ -220,6 +222,42 @@ public void testGetChannelRefInitialization() { assertEquals(1, gcpChannel.channelRefs.size()); } + @Test + public void testGetChannelRefInitializationWithMinSize() throws InterruptedException { + resetGcpChannel(); + GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() + .setMaxSize(5) + .setMinSize(2) + .build(); + GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(poolOptions) + .build(); + gcpChannel = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withOptions(options) + .build(); + // Should have 2 channels since the beginning. + assertThat(gcpChannel.channelRefs.size()).isEqualTo(2); + TimeUnit.MILLISECONDS.sleep(50); + // The connection establishment must have been started on these two channels. + assertThat(gcpChannel.getState(false)) + .isAnyOf( + ConnectivityState.CONNECTING, + ConnectivityState.READY, + ConnectivityState.TRANSIENT_FAILURE); + assertThat(gcpChannel.channelRefs.get(0).getChannel().getState(false)) + .isAnyOf( + ConnectivityState.CONNECTING, + ConnectivityState.READY, + ConnectivityState.TRANSIENT_FAILURE); + assertThat(gcpChannel.channelRefs.get(1).getChannel().getState(false)) + .isAnyOf( + ConnectivityState.CONNECTING, + ConnectivityState.READY, + ConnectivityState.TRANSIENT_FAILURE); + } + @Test public void testGetChannelRefPickUpSmallest() { // All channels have max number of streams From a2de65dfb76fa209a9785044ae7e207fc3923487 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 30 Jun 2022 13:20:57 -0700 Subject: [PATCH 75/87] Add MultiEndpoint --- .../cloud/grpc/multiendpoint/Endpoint.java | 78 ++++ .../grpc/multiendpoint/MultiEndpoint.java | 201 ++++++++++ .../grpc/multiendpoint/MultiEndpointTest.java | 360 ++++++++++++++++++ 3 files changed, 639 insertions(+) create mode 100644 grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java create mode 100644 grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java create mode 100644 grpc-gcp/src/test/java/com/google/cloud/grpc/multiendpoint/MultiEndpointTest.java diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java new file mode 100644 index 00000000..d6f4debd --- /dev/null +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java @@ -0,0 +1,78 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.grpc.multiendpoint; + +import com.google.errorprone.annotations.CheckReturnValue; +import java.util.concurrent.ScheduledFuture; + +/** + * Endpoint holds an endpoint's state, priority and a future of upcoming state change. + */ +@CheckReturnValue +final class Endpoint { + + /** + * Holds a state of an endpoint. + */ + public enum EndpointState { + UNAVAILABLE, + AVAILABLE, + RECOVERING, + } + + private final String id; + private EndpointState state; + private int priority; + private ScheduledFuture changeStateFuture; + + public Endpoint(String id, EndpointState state, int priority) { + this.id = id; + this.priority = priority; + this.state = state; + } + + public String getId() { + return id; + } + + public EndpointState getState() { + return state; + } + + public int getPriority() { + return priority; + } + + public void setState(EndpointState state) { + this.state = state; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public synchronized void setChangeStateFuture(ScheduledFuture future) { + resetStateChangeFuture(); + this.changeStateFuture = future; + } + + public synchronized void resetStateChangeFuture() { + if (changeStateFuture != null) { + changeStateFuture.cancel(true); + } + } +} diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java new file mode 100644 index 00000000..c98f31d1 --- /dev/null +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java @@ -0,0 +1,201 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.grpc.multiendpoint; + +import static java.util.Comparator.comparingInt; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.cloud.grpc.multiendpoint.Endpoint.EndpointState; +import com.google.common.base.Preconditions; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +/** + * MultiEndpoint holds a list of endpoints, tracks their availability and defines the current + * endpoint. An endpoint has a priority defined by its position in the list (first item has top + * priority). MultiEndpoint returns top priority endpoint that is available as current. If no + * endpoint is available, MultiEndpoint returns the top priority endpoint. + * + *

Sometimes switching between endpoints can be costly, and it is worth to wait for some time + * after current endpoint becomes unavailable. For this case, use {@link + * Builder#withRecoveryTimeout} to set the recovery timeout. MultiEndpoint will keep the current + * endpoint for up to recovery timeout after it became unavailable to give it some time to recover. + * + *

The list of endpoints can be changed at any time with {@link #setEndpoints} method. + * MultiEndpoint will preserve endpoints' state and update their priority according to their new + * positions. + * + *

The initial state of endpoint is "unavailable" or "recovering" if using recovery timeout. + */ +@CheckReturnValue +public final class MultiEndpoint { + @GuardedBy("this") + private final Map endpointsMap = new HashMap<>(); + + @GuardedBy("this") + private String currentId; + + private final Duration recoveryTimeout; + private final boolean recoveryEnabled; + + private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); + + private MultiEndpoint(Builder builder) { + this.recoveryTimeout = builder.recoveryTimeout; + this.recoveryEnabled = + !builder.recoveryTimeout.isNegative() && !builder.recoveryTimeout.isZero(); + this.setEndpoints(builder.endpoints); + } + + /** Builder for MultiEndpoint. */ + public static final class Builder { + private final List endpoints; + private Duration recoveryTimeout = Duration.ZERO; + + public Builder(List endpoints) { + Preconditions.checkNotNull(endpoints); + Preconditions.checkArgument(!endpoints.isEmpty(), "Endpoints list must not be empty."); + this.endpoints = endpoints; + } + + /** + * MultiEndpoint will keep the current endpoint for up to recovery timeout after it became + * unavailable to give it some time to recover. + */ + public Builder withRecoveryTimeout(Duration timeout) { + Preconditions.checkNotNull(timeout); + this.recoveryTimeout = timeout; + return this; + } + + public MultiEndpoint build() { + return new MultiEndpoint(this); + } + } + + /** + * Returns current endpoint id. + * + *

Note that the read is not synchronized and in case of a race condition there is a chance of + * getting an outdated current id. + */ + @SuppressWarnings("GuardedBy") + public String getCurrentId() { + return currentId; + } + + private synchronized void setEndpointStateInternal(String endpointId, EndpointState state) { + Endpoint endpoint = endpointsMap.get(endpointId); + if (endpoint != null) { + endpoint.setState(state); + maybeFallback(); + } + } + + /** Inform MultiEndpoint when an endpoint becomes available or unavailable. */ + public synchronized void setEndpointAvailable(String endpointId, boolean available) { + setEndpointState(endpointId, available ? EndpointState.AVAILABLE : EndpointState.UNAVAILABLE); + } + + private synchronized void setEndpointState(String endpointId, EndpointState state) { + Preconditions.checkNotNull(state); + Endpoint endpoint = endpointsMap.get(endpointId); + if (endpoint == null) { + return; + } + // If we allow some recovery time. + if (EndpointState.UNAVAILABLE.equals(state) && recoveryEnabled) { + endpoint.setState(EndpointState.RECOVERING); + ScheduledFuture future = + executor.schedule( + () -> setEndpointStateInternal(endpointId, EndpointState.UNAVAILABLE), + recoveryTimeout.toMillis(), + MILLISECONDS); + endpoint.setChangeStateFuture(future); + return; + } + endpoint.resetStateChangeFuture(); + endpoint.setState(state); + maybeFallback(); + } + + /** + * Provide an updated list of endpoints to MultiEndpoint. + * + *

MultiEndpoint will preserve current endpoints' state and update their priority according to + * their new positions. + */ + public synchronized void setEndpoints(List endpoints) { + Preconditions.checkNotNull(endpoints); + Preconditions.checkArgument(!endpoints.isEmpty(), "Endpoints list must not be empty."); + + // Remove obsolete endpoints. + endpointsMap.keySet().retainAll(endpoints); + + // Add new endpoints and update priority. + int priority = 0; + for (String endpointId : endpoints) { + Endpoint existingEndpoint = endpointsMap.get(endpointId); + if (existingEndpoint != null) { + existingEndpoint.setPriority(priority++); + continue; + } + EndpointState newState = + recoveryEnabled ? EndpointState.RECOVERING : EndpointState.UNAVAILABLE; + Endpoint newEndpoint = new Endpoint(endpointId, newState, priority++); + if (recoveryEnabled) { + ScheduledFuture future = + executor.schedule( + () -> setEndpointStateInternal(endpointId, EndpointState.UNAVAILABLE), + recoveryTimeout.toMillis(), + MILLISECONDS); + newEndpoint.setChangeStateFuture(future); + } + endpointsMap.put(endpointId, newEndpoint); + } + + maybeFallback(); + } + + private synchronized void maybeFallback() { + Optional topEndpoint = + endpointsMap.values().stream() + .filter((c) -> c.getState().equals(EndpointState.AVAILABLE)) + .min(comparingInt(Endpoint::getPriority)); + + Endpoint current = endpointsMap.get(currentId); + if (current != null && current.getState().equals(EndpointState.RECOVERING)) { + // Keep recovering endpoint as current unless a higher priority endpoint became available. + if (!topEndpoint.isPresent() || topEndpoint.get().getPriority() >= current.getPriority()) { + return; + } + } + + if (!topEndpoint.isPresent() && current == null) { + topEndpoint = endpointsMap.values().stream().min(comparingInt(Endpoint::getPriority)); + } + + topEndpoint.ifPresent(endpoint -> currentId = endpoint.getId()); + } +} diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/multiendpoint/MultiEndpointTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/multiendpoint/MultiEndpointTest.java new file mode 100644 index 00000000..7dc0ac0b --- /dev/null +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/multiendpoint/MultiEndpointTest.java @@ -0,0 +1,360 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.grpc.multiendpoint; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.client.util.Sleeper; +import com.google.common.collect.ImmutableList; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for MultiEndpoint. + */ +@RunWith(JUnit4.class) +public final class MultiEndpointTest { + private final List threeEndpoints = + new ArrayList<>(ImmutableList.of("first", "second", "third")); + + private final List fourEndpoints = + new ArrayList<>(ImmutableList.of("four", "first", "third", "second")); + + private static final long RECOVERY_MS = 1000; + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + private MultiEndpoint initPlain(List endpoints) { + return new MultiEndpoint.Builder(endpoints).build(); + } + + private MultiEndpoint initWithRecovery(List endpoints, long recoveryTimeOut) { + return new MultiEndpoint.Builder(endpoints) + .withRecoveryTimeout(Duration.ofMillis(recoveryTimeOut)) + .build(); + } + + @Test + public void initPlain_raisesErrorWhenEmptyEndpoints() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Endpoints list must not be empty."); + initPlain(ImmutableList.of()); + } + + @Test + public void initWithRecovery_raisesErrorWhenEmptyEndpoints() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Endpoints list must not be empty."); + initWithRecovery(ImmutableList.of(), RECOVERY_MS); + } + + @Test + public void getCurrent_returnsTopPriorityAvailableEndpointWithoutRecovery() { + MultiEndpoint multiEndpoint = initPlain(threeEndpoints); + + // Returns first after creation. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // Second becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + + // Second is the current as the only available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Third becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), true); + + // Second is still the current because it has higher priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // First becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(0), true); + + // First becomes the current because it has higher priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // Second becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), false); + + // Second becoming unavailable should not affect the current first. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // First becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(0), false); + + // Third becomes the current as the only remaining available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(2)); + + // Third becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), false); + + // After all endpoints became unavailable the multiEndpoint sticks to the last used endpoint. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(2)); + } + + @Test + public void getCurrent_returnsTopPriorityAvailableEndpointWithRecovery() + throws InterruptedException { + MultiEndpoint multiEndpoint = initWithRecovery(threeEndpoints, RECOVERY_MS); + + // Returns first after creation. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // Second becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + + // First is still the current to allow it to become available within recovery timeout. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Second becomes current as an available endpoint with top priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Third becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), true); + + // Second is still the current because it has higher priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Second becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), false); + + // Second is still current, allowing upto recoveryTimeout to recover. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Halfway through recovery timeout the second recovers. + Sleeper.DEFAULT.sleep(RECOVERY_MS / 2); + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + + // Second is the current. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // After the initial recovery timeout, the second is still current. + Sleeper.DEFAULT.sleep(RECOVERY_MS / 2 + 100); + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Second becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), false); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Changes to an available endpoint -- third. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(2)); + + // First becomes available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(0), true); + + // First becomes current immediately. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // First becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(0), false); + + // First is still current, allowing upto recoveryTimeout to recover. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(0)); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Changes to an available endpoint -- third. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(2)); + + // Third becomes unavailable + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), false); + + // Third is still current, allowing upto recoveryTimeout to recover. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(2)); + + // Halfway through recovery timeout the second becomes available. + // Sleeper.defaultSleeper().sleep(Duration.ofMillis(RECOVERY_MS - 100)); + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + + // Second becomes current immediately. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // Second becomes unavailable. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), false); + + // Second is still current, allowing upto recoveryTimeout to recover. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // After all endpoints became unavailable the multiEndpoint sticks to the last used endpoint. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(threeEndpoints.get(1)); + } + + @Test + public void setEndpoints_raisesErrorWhenEmptyEndpoints() { + MultiEndpoint multiEndpoint = initPlain(threeEndpoints); + expectedEx.expect(IllegalArgumentException.class); + multiEndpoint.setEndpoints(ImmutableList.of()); + } + + @Test + public void setEndpoints_updatesEndpoints() { + MultiEndpoint multiEndpoint = initPlain(threeEndpoints); + multiEndpoint.setEndpoints(fourEndpoints); + + // "first" which is now under index 1 still current because no other available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(1)); + } + + @Test + public void setEndpoints_updatesEndpointsWithRecovery() { + MultiEndpoint multiEndpoint = initWithRecovery(threeEndpoints, RECOVERY_MS); + multiEndpoint.setEndpoints(fourEndpoints); + + // "first" which is now under index 1 still current because no other available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(1)); + } + + @Test + public void setEndpoints_updatesEndpointsPreservingStates() { + MultiEndpoint multiEndpoint = initPlain(threeEndpoints); + + // Second is available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + multiEndpoint.setEndpoints(fourEndpoints); + + // "second" which is now under index 3 still must remain available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(3)); + } + + @Test + public void setEndpoints_updatesEndpointsPreservingStatesWithRecovery() + throws InterruptedException { + MultiEndpoint multiEndpoint = initWithRecovery(threeEndpoints, RECOVERY_MS); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Second is available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + multiEndpoint.setEndpoints(fourEndpoints); + + // "second" which is now under index 3 still must remain available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(3)); + } + + @Test + public void setEndpoints_updatesEndpointsSwitchToTopPriorityAvailable() { + MultiEndpoint multiEndpoint = initPlain(threeEndpoints); + + // Second and third is available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), true); + + multiEndpoint.setEndpoints(fourEndpoints); + + // "third" which is now under index 2 must become current, because "second" has lower priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(2)); + } + + @Test + public void setEndpoints_updatesEndpointsSwitchToTopPriorityAvailableWithRecovery() + throws InterruptedException { + MultiEndpoint multiEndpoint = initWithRecovery(threeEndpoints, RECOVERY_MS); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Second and third is available. + multiEndpoint.setEndpointAvailable(threeEndpoints.get(1), true); + multiEndpoint.setEndpointAvailable(threeEndpoints.get(2), true); + + multiEndpoint.setEndpoints(fourEndpoints); + + // "third" which is now under index 2 must become current, because "second" has lower priority. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(2)); + } + + @Test + public void setEndpoints_updatesEndpointsRemovesOnlyActiveEndpoint() { + List extraEndpoints = new ArrayList<>(threeEndpoints); + extraEndpoints.add("extra"); + MultiEndpoint multiEndpoint = initPlain(extraEndpoints); + + // Extra is available. + multiEndpoint.setEndpointAvailable("extra", true); + + // Extra is removed. + multiEndpoint.setEndpoints(fourEndpoints); + + // "four" which is under index 0 must become current, because no endpoints available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(0)); + } + + @Test + public void setEndpoints_updatesEndpointsRemovesOnlyActiveEndpointWithRecovery() + throws InterruptedException { + List extraEndpoints = new ArrayList<>(threeEndpoints); + extraEndpoints.add("extra"); + MultiEndpoint multiEndpoint = initWithRecovery(extraEndpoints, RECOVERY_MS); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Extra is available. + multiEndpoint.setEndpointAvailable("extra", true); + + // Extra is removed. + multiEndpoint.setEndpoints(fourEndpoints); + + // "four" which is under index 0 must become current, because no endpoints available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(0)); + } + + @Test + public void setEndpoints_recoveringEndpointGetsRemoved() throws InterruptedException { + List extraEndpoints = new ArrayList<>(threeEndpoints); + extraEndpoints.add("extra"); + MultiEndpoint multiEndpoint = initWithRecovery(extraEndpoints, RECOVERY_MS); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // Extra is available. + multiEndpoint.setEndpointAvailable("extra", true); + + // Extra is recovering. + multiEndpoint.setEndpointAvailable("extra", false); + + // Extra is removed. + multiEndpoint.setEndpoints(fourEndpoints); + + // "four" which is under index 0 must become current, because no endpoints available. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(0)); + + // After recovery timeout has passed. + Sleeper.DEFAULT.sleep(RECOVERY_MS + 100); + + // "four" is still current. + assertThat(multiEndpoint.getCurrentId()).isEqualTo(fourEndpoints.get(0)); + } +} From 489efd9342581c8ec9be86a60649c37c86504714 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 30 Jun 2022 15:45:40 -0700 Subject: [PATCH 76/87] Allow GcpManagedChannel state change notifications. --- .../google/cloud/grpc/GcpManagedChannel.java | 39 ++++++++++++++++- .../cloud/grpc/GcpManagedChannelTest.java | 42 ++++++++++++++++++- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 64c7cca2..70c45ea9 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -25,6 +25,7 @@ import com.google.cloud.grpc.proto.ApiConfig; import com.google.cloud.grpc.proto.MethodConfig; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.base.Joiner; import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.protobuf.Descriptors.FieldDescriptor; @@ -48,11 +49,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -92,6 +95,10 @@ public class GcpManagedChannel extends ManagedChannel { @VisibleForTesting final List channelRefs = new CopyOnWriteArrayList<>(); + private final ExecutorService stateNotificationExecutor = Executors.newCachedThreadPool( + new ThreadFactoryBuilder().setNameFormat("gcp-mc-state-notifications-%d").build()); + private List stateChangeCallbacks = Collections.synchronizedList(new LinkedList<>()); + // Metrics configuration. private MetricRegistry metricRegistry; private final List labelKeys = new ArrayList<>(); @@ -872,6 +879,15 @@ private void recordUnresponsiveDetection(long nanos, long dropCount) { } } + @Override + public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) { + if (!getState(false).equals(source)) { + stateNotificationExecutor.execute(callback); + return; + } + stateChangeCallbacks.add(callback); + } + /** * ChannelStateMonitor subscribes to channel's state changes and informs {@link GcpManagedChannel} * on any new state. This monitor allows to detect when a channel is not ready and temporarily @@ -919,7 +935,14 @@ public void run() { } } + private synchronized void executeStateChangeCallbacks() { + List callbacksToTrigger = stateChangeCallbacks; + stateChangeCallbacks = new LinkedList<>(); + callbacksToTrigger.forEach(stateNotificationExecutor::execute); + } + void processChannelStateChange(int channelId, ConnectivityState state) { + executeStateChangeCallbacks(); if (!fallbackEnabled) { return; } @@ -1164,6 +1187,9 @@ public ManagedChannel shutdownNow() { if (logMetricService != null && !logMetricService.isTerminated()) { logMetricService.shutdownNow(); } + if (!stateNotificationExecutor.isTerminated()) { + stateNotificationExecutor.shutdownNow(); + } return this; } @@ -1176,6 +1202,7 @@ public ManagedChannel shutdown() { if (logMetricService != null) { logMetricService.shutdown(); } + stateNotificationExecutor.shutdown(); return this; } @@ -1197,6 +1224,11 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE //noinspection ResultOfMethodCallIgnored logMetricService.awaitTermination(awaitTimeNanos, NANOSECONDS); } + awaitTimeNanos = endTimeNanos - System.nanoTime(); + if (awaitTimeNanos > 0) { + //noinspection ResultOfMethodCallIgnored + stateNotificationExecutor.awaitTermination(awaitTimeNanos, NANOSECONDS); + } return isTerminated(); } @@ -1210,7 +1242,7 @@ public boolean isShutdown() { if (logMetricService != null) { return logMetricService.isShutdown(); } - return true; + return stateNotificationExecutor.isShutdown(); } @Override @@ -1223,12 +1255,15 @@ public boolean isTerminated() { if (logMetricService != null) { return logMetricService.isTerminated(); } - return true; + return stateNotificationExecutor.isTerminated(); } /** Get the current connectivity state of the channel pool. */ @Override public ConnectivityState getState(boolean requestConnection) { + if (requestConnection && getNumberOfChannels() == 0) { + createNewChannel(); + } int ready = 0; int idle = 0; int connecting = 0; diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 417ced1e..56c1e946 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -55,7 +55,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -100,7 +102,7 @@ private Level lastLogLevel(int nthFromLast) { private final Handler testLogHandler = new Handler() { @Override - public void publish(LogRecord record) { + public synchronized void publish(LogRecord record) { logRecords.add(record); } @@ -1225,6 +1227,44 @@ public void testUnresponsiveDetection() throws InterruptedException { poolIndex + ": stat: " + GcpMetricsConstants.METRIC_NUM_UNRESPONSIVE_DETECTIONS + " = 0"); } + @Test + public void testStateNotifications() throws InterruptedException { + final AtomicBoolean immediateCallbackCalled = new AtomicBoolean(); + // Test callback is called when state doesn't match. + gcpChannel.notifyWhenStateChanged(ConnectivityState.SHUTDOWN, () -> + immediateCallbackCalled.set(true)); + + TimeUnit.MILLISECONDS.sleep(1); + + assertThat(immediateCallbackCalled.get()).isTrue(); + + // Subscribe for notification when leaving IDLE state. + final AtomicReference newState = new AtomicReference<>(); + + final Runnable callback = new Runnable() { + @Override + public void run() { + ConnectivityState state = gcpChannel.getState(false); + newState.set(state); + if (state.equals(ConnectivityState.IDLE)) { + gcpChannel.notifyWhenStateChanged(ConnectivityState.IDLE, this); + } + } + }; + + gcpChannel.notifyWhenStateChanged(ConnectivityState.IDLE, callback); + + // Init connection to move out of the IDLE state. + ConnectivityState currentState = gcpChannel.getState(true); + // Make sure it was IDLE; + assertThat(currentState).isEqualTo(ConnectivityState.IDLE); + + TimeUnit.MILLISECONDS.sleep(5); + + assertThat(newState.get()) + .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE); + } + static class FakeManagedChannel extends ManagedChannel { private ConnectivityState state = ConnectivityState.IDLE; private Runnable stateCallback; From bc260b86eab9c198824fcb2635b3c2ad2f89612f Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 30 Jun 2022 18:29:22 -0700 Subject: [PATCH 77/87] Remove race conditions when creating new channels. --- .../google/cloud/grpc/GcpManagedChannel.java | 56 +++++++++++--- .../cloud/grpc/GcpManagedChannelTest.java | 77 ++++++++++++++++++- 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java index 70c45ea9..4354cce7 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java @@ -990,10 +990,12 @@ protected ChannelRef getChannelRefForBind() { ChannelRef channelRef; if (options.getChannelPoolOptions() != null && options.getChannelPoolOptions().isUseRoundRobinOnBind()) { channelRef = getChannelRefRoundRobin(); + logger.finest(log( + "Channel %d picked for bind operation using round-robin.", channelRef.getId())); } else { channelRef = getChannelRef(null); + logger.finest(log("Channel %d picked for bind operation.", channelRef.getId())); } - logger.finest(log("Channel %d picked for bind operation.", channelRef.getId())); return channelRef; } @@ -1084,6 +1086,35 @@ private synchronized ChannelRef createNewChannel() { return channelRef; } + // Returns first newly created channel or null if there are already some channels in the pool. + @Nullable + private ChannelRef createFirstChannel() { + if (!channelRefs.isEmpty()) { + return null; + } + synchronized (this) { + if (channelRefs.isEmpty()) { + return createNewChannel(); + } + } + return null; + } + + // Creates new channel if maxSize is not reached. + // Returns new channel or null. + @Nullable + private ChannelRef tryCreateNewChannel() { + if (channelRefs.size() >= maxSize) { + return null; + } + synchronized (this) { + if (channelRefs.size() < maxSize) { + return createNewChannel(); + } + } + return null; + } + /** * Pick a {@link ChannelRef} (and create a new one if necessary). If notReadyFallbackEnabled is * true in the {@link GcpResiliencyOptions} then instead of a channel in a non-READY state another @@ -1091,8 +1122,9 @@ private synchronized ChannelRef createNewChannel() { * be provided if available. */ private ChannelRef pickLeastBusyChannel(boolean forFallback) { - if (channelRefs.isEmpty()) { - return createNewChannel(); + ChannelRef first = createFirstChannel(); + if (first != null) { + return first; } // Pick the least busy channel and the least busy ready and not overloaded channel (this could @@ -1118,17 +1150,23 @@ private ChannelRef pickLeastBusyChannel(boolean forFallback) { if (!fallbackEnabled) { if (channelRefs.size() < maxSize && minStreams >= maxConcurrentStreamsLowWatermark) { - return createNewChannel(); + ChannelRef newChannel = tryCreateNewChannel(); + if (newChannel != null) { + return newChannel; + } } return channelCandidate; } if (channelRefs.size() < maxSize && readyMinStreams >= maxConcurrentStreamsLowWatermark) { - if (!forFallback && readyCandidate == null) { - logger.finest(log("Fallback to newly created channel")); - fallbacksSucceeded.incrementAndGet(); + ChannelRef newChannel = tryCreateNewChannel(); + if (newChannel != null) { + if (!forFallback && readyCandidate == null) { + logger.finest(log("Fallback to newly created channel %d", newChannel.getId())); + fallbacksSucceeded.incrementAndGet(); + } + return newChannel; } - return createNewChannel(); } if (readyCandidate != null) { @@ -1262,7 +1300,7 @@ public boolean isTerminated() { @Override public ConnectivityState getState(boolean requestConnection) { if (requestConnection && getNumberOfChannels() == 0) { - createNewChannel(); + createFirstChannel(); } int ready = 0; int idle = 0; diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java index 56c1e946..9af0e88e 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java @@ -379,9 +379,10 @@ public void testGetChannelRefWithFallback() { assertEquals(2, pool.getNumberOfChannels()); // This was a fallback from non-ready channel 0 to the newly created channel 1. assertThat(logRecords.size()).isEqualTo(logCount + 3); - assertThat(lastLogMessage(3)).isEqualTo( - poolIndex + ": Fallback to newly created channel"); - assertThat(lastLogLevel(3)).isEqualTo(Level.FINEST); + logRecords.forEach(logRecord -> System.out.println(logRecord.getMessage())); + assertThat(lastLogMessage()).isEqualTo( + poolIndex + ": Fallback to newly created channel 1"); + assertThat(lastLogLevel()).isEqualTo(Level.FINEST); assertFallbacksMetric(fakeRegistry, 1, 0); // Adding one active stream to channel 1. @@ -1265,6 +1266,76 @@ public void run() { .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE); } + @Test + public void testParallelGetChannelRefWontExceedMaxSize() throws InterruptedException { + resetGcpChannel(); + GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() + .setMaxSize(2) + .setConcurrentStreamsLowWatermark(0) + .build(); + GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(poolOptions) + .build(); + gcpChannel = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withOptions(options) + .build(); + + assertThat(gcpChannel.getNumberOfChannels()).isEqualTo(0); + assertThat(gcpChannel.getStreamsLowWatermark()).isEqualTo(0); + + for (int i = 0; i < gcpChannel.getMaxSize() - 1; i++) { + gcpChannel.getChannelRef(null); + } + + assertThat(gcpChannel.getNumberOfChannels()).isEqualTo(gcpChannel.getMaxSize() - 1); + + Runnable requestChannel = () -> gcpChannel.getChannelRef(null); + + int requestCount = gcpChannel.getMaxSize() * 3; + ExecutorService exec = Executors.newFixedThreadPool(requestCount); + for (int i = 0; i < requestCount; i++) { + exec.execute(requestChannel); + } + exec.shutdown(); + exec.awaitTermination(100, TimeUnit.MILLISECONDS); + + assertThat(gcpChannel.getNumberOfChannels()).isEqualTo(gcpChannel.getMaxSize()); + } + + @Test + public void testParallelGetChannelRefWontExceedMaxSizeFromTheStart() throws InterruptedException { + resetGcpChannel(); + GcpChannelPoolOptions poolOptions = GcpChannelPoolOptions.newBuilder() + .setMaxSize(2) + .setConcurrentStreamsLowWatermark(0) + .build(); + GcpManagedChannelOptions options = GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(poolOptions) + .build(); + gcpChannel = + (GcpManagedChannel) + GcpManagedChannelBuilder.forDelegateBuilder(builder) + .withOptions(options) + .build(); + + assertThat(gcpChannel.getNumberOfChannels()).isEqualTo(0); + assertThat(gcpChannel.getStreamsLowWatermark()).isEqualTo(0); + + Runnable requestChannel = () -> gcpChannel.getChannelRef(null); + + int requestCount = gcpChannel.getMaxSize() * 3; + ExecutorService exec = Executors.newFixedThreadPool(requestCount); + for (int i = 0; i < requestCount; i++) { + exec.execute(requestChannel); + } + exec.shutdown(); + exec.awaitTermination(100, TimeUnit.MILLISECONDS); + + assertThat(gcpChannel.getNumberOfChannels()).isEqualTo(gcpChannel.getMaxSize()); + } + static class FakeManagedChannel extends ManagedChannel { private ConnectivityState state = ConnectivityState.IDLE; private Runnable stateCallback; From 4f675799198e8416942f13dec8887a2492f3959b Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 30 Jun 2022 18:30:50 -0700 Subject: [PATCH 78/87] Add minSize to options toString, reorder options in toString. --- .../google/cloud/grpc/GcpManagedChannelOptions.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java index 9f66d721..f6c4c557 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java @@ -20,6 +20,7 @@ import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.MetricRegistry; + import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -64,10 +65,10 @@ public GcpResiliencyOptions getResiliencyOptions() { @Override public String toString() { return String.format( - "{channelPoolOptions: %s, metricsOptions: %s, resiliencyOptions: %s}", + "{channelPoolOptions: %s, resiliencyOptions: %s, metricsOptions: %s}", getChannelPoolOptions(), - getMetricsOptions(), - getResiliencyOptions() + getResiliencyOptions(), + getMetricsOptions() ); } @@ -208,8 +209,9 @@ public boolean isUseRoundRobinOnBind() { @Override public String toString() { return String.format( - "{maxSize: %d, concurrentStreamsLowWatermark: %d, useRoundRobinOnBind: %s}", + "{maxSize: %d, minSize: %d, concurrentStreamsLowWatermark: %d, useRoundRobinOnBind: %s}", getMaxSize(), + getMinSize(), getConcurrentStreamsLowWatermark(), isUseRoundRobinOnBind() ); From 28fcaf4f62a86e361cd0f4f5aaeac31cb5c036ba Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 30 Jun 2022 20:04:50 -0700 Subject: [PATCH 79/87] Add multi-endpoint feature. --- grpc-gcp/build.gradle | 1 + .../cloud/grpc/GcpMultiEndpointChannel.java | 402 ++++++++++++++++++ .../cloud/grpc/GcpMultiEndpointOptions.java | 170 ++++++++ .../cloud/grpc/SpannerIntegrationTest.java | 320 +++++++++++++- 4 files changed, 873 insertions(+), 20 deletions(-) create mode 100644 grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java create mode 100644 grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java diff --git a/grpc-gcp/build.gradle b/grpc-gcp/build.gradle index 367379e4..32e37965 100644 --- a/grpc-gcp/build.gradle +++ b/grpc-gcp/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" implementation "io.grpc:grpc-stub:${grpcVersion}" implementation "io.opencensus:opencensus-api:${opencensusVersion}" + implementation "com.google.api:api-common:2.1.5" compileOnly "org.apache.tomcat:annotations-api:6.0.53" // necessary for Java 9+ diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java new file mode 100644 index 00000000..a1b1276e --- /dev/null +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java @@ -0,0 +1,402 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.grpc; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions; +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions; +import com.google.cloud.grpc.multiendpoint.MultiEndpoint; +import com.google.cloud.grpc.proto.ApiConfig; +import com.google.common.base.Preconditions; +import io.grpc.CallOptions; +import io.grpc.ClientCall; +import io.grpc.ClientCall.Listener; +import io.grpc.ConnectivityState; +import io.grpc.Grpc; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * The purpose of GcpMultiEndpointChannel is twofold: + * + *

    + *
  1. Fallback to an alternative endpoint (host:port) of a gRPC service when the original + * endpoint is completely unavailable. + *
  2. Be able to route an RPC call to a specific group of endpoints. + *
+ * + *

A group of endpoints is called a {@link MultiEndpoint} and is essentially a list of endpoints + * where priority is defined by the position in the list with the first endpoint having top + * priority. A MultiEndpoint tracks endpoints' availability. When a MultiEndpoint is picked for an + * RPC call, it picks the top priority endpoint that is currently available. More information on + * the {@link MultiEndpoint} class. + * + *

GcpMultiEndpointChannel can have one or more MultiEndpoint identified by its name -- arbitrary + * string provided in the {@link GcpMultiEndpointOptions} when configuring MultiEndpoints. This name + * can be used to route an RPC call to this MultiEndpoint by setting the {@link #ME_KEY} key value + * of the RPC {@link CallOptions}. + * + *

GcpMultiEndpointChannel receives a list of GcpMultiEndpointOptions for initial configuration. + * An updated configuration can be provided at any time later using + * {@link GcpMultiEndpointChannel#setMultiEndpoints(List)}. The first item in the + * GcpMultiEndpointOptions list defines the default MultiEndpoint that will be used when no + * MultiEndpoint name is provided with an RPC call. + * + *

Example configuration: + *

    + *
  • + * MultiEndpoint named "default" with endpoints: + *
      + *
    1. service.example.com:443
      2. + *
    2. service-fallback.example.com:443
      3. + *
    + *
    • + *
  • + * MultiEndpoint named "read" with endpoints: + *
      + *
    1. ro-service.example.com:443
      2. + *
    2. service-fallback.example.com:443
      3. + *
    3. service.example.com:443
      4. + *
    + *
    • + *
+ * + *

Let's assume we have a service with read and write operations and the following backends: + *

    + *
  • service.example.com -- the main set of backends supporting all operations
    • + *
  • service-fallback.example.com -- read-write replica supporting all operations
    • + *
  • ro-service.example.com -- read-only replica supporting only read operations
    • + *
+ * + *

With the configuration above GcpMultiEndpointChannel will use the "default" MultiEndpoint by + * default. It means that RPC calls by default will use the main endpoint and if it is not available + * then the read-write replica. + * + *

To offload some read calls to the read-only replica we can specify "read" MultiEndpoint in + * the CallOptions. Then these calls will use the read-only replica endpoint and if it is not + * available then the read-write replica and if it is also not available then the main endpoint. + * + *

GcpMultiEndpointChannel creates a {@link GcpManagedChannel} channel pool for every unique + * endpoint. For the example above three channel pools will be created. + */ +public class GcpMultiEndpointChannel extends ManagedChannel { + + public static final CallOptions.Key ME_KEY = CallOptions.Key.create("MultiEndpoint"); + private final LabelKey endpointKey = + LabelKey.create("endpoint", "Endpoint address."); + private final Map multiEndpoints = new ConcurrentHashMap<>(); + private MultiEndpoint defaultMultiEndpoint; + private final ApiConfig apiConfig; + private final GcpManagedChannelOptions gcpManagedChannelOptions; + + private final Map pools = new ConcurrentHashMap<>(); + + /** + * Constructor for {@link GcpMultiEndpointChannel}. + * + * @param meOptions list of MultiEndpoint configurations. + * @param apiConfig the ApiConfig object for configuring GcpManagedChannel. + * @param gcpManagedChannelOptions the options for GcpManagedChannel. + */ + public GcpMultiEndpointChannel( + List meOptions, + ApiConfig apiConfig, + GcpManagedChannelOptions gcpManagedChannelOptions) { + this.apiConfig = apiConfig; + this.gcpManagedChannelOptions = gcpManagedChannelOptions; + setMultiEndpoints(meOptions); + } + + private class EndpointStateMonitor implements Runnable { + + private final ManagedChannel channel; + private final String endpoint; + + private EndpointStateMonitor(ManagedChannel channel, String endpoint) { + this.endpoint = endpoint; + this.channel = channel; + run(); + } + + @Override + public void run() { + if (channel == null) { + return; + } + ConnectivityState newState = checkPoolState(channel, endpoint); + if (newState != ConnectivityState.SHUTDOWN) { + channel.notifyWhenStateChanged(newState, this); + } + } + } + + // Checks and returns channel pool state. Also notifies all MultiEndpoints of the pool state. + private ConnectivityState checkPoolState(ManagedChannel channel, String endpoint) { + ConnectivityState state = channel.getState(false); + // Update endpoint state in all multiendpoints. + for (MultiEndpoint me : multiEndpoints.values()) { + me.setEndpointAvailable(endpoint, state.equals(ConnectivityState.READY)); + } + return state; + } + + private GcpManagedChannelOptions prepareGcpManagedChannelConfig( + GcpManagedChannelOptions gcpOptions, String endpoint) { + final GcpMetricsOptions.Builder metricsOptions = GcpMetricsOptions.newBuilder( + gcpOptions.getMetricsOptions() + ); + + final List labelKeys = new ArrayList<>(metricsOptions.build().getLabelKeys()); + final List labelValues = new ArrayList<>(metricsOptions.build().getLabelValues()); + + labelKeys.add(endpointKey); + labelValues.add(LabelValue.create(endpoint)); + + // Make sure the pool will have at least 1 channel always connected. If maximum size > 1 then we + // want at least 2 channels or square root of maximum channels whichever is larger. + // Do not override if minSize is already specified as > 0. + final GcpChannelPoolOptions.Builder poolOptions = GcpChannelPoolOptions.newBuilder( + gcpOptions.getChannelPoolOptions() + ); + if (poolOptions.build().getMinSize() < 1) { + int minSize = Math.min(2, poolOptions.build().getMaxSize()); + minSize = Math.max(minSize, ((int) Math.sqrt(poolOptions.build().getMaxSize()))); + poolOptions.setMinSize(minSize); + } + + return GcpManagedChannelOptions.newBuilder(gcpOptions) + .withChannelPoolOptions(poolOptions.build()) + .withMetricsOptions(metricsOptions.withLabels(labelKeys, labelValues).build()) + .build(); + } + + /** + * Update the list of MultiEndpoint configurations. + * + *

MultiEndpoints are matched with the current ones by name. + *

    + *
  • If a current MultiEndpoint is missing in the updated list, the MultiEndpoint will be + * removed. + *
  • A new MultiEndpoint will be created for every new name in the list. + *
  • For an existing MultiEndpoint only its endpoints will be updated (no recovery timeout + * change). + *
+ * + *

Endpoints are matched by the endpoint address (usually in the form of address:port). + *

    + *
  • If an existing endpoint is not used by any MultiEndpoint in the updated list, then the + * channel poll for this endpoint will be shutdown. + *
  • A channel pool will be created for every new endpoint. + *
  • For an existing endpoint nothing will change (the channel pool will not be re-created, thus + * no channel credentials change, nor channel configurator change). + *
+ */ + public void setMultiEndpoints(List meOptions) { + Preconditions.checkNotNull(meOptions); + Preconditions.checkArgument(!meOptions.isEmpty(), "MultiEndpoints list is empty"); + Set currentMultiEndpoints = new HashSet<>(); + Set currentEndpoints = new HashSet<>(); + meOptions.forEach(options -> { + currentMultiEndpoints.add(options.getName()); + // Create or update MultiEndpoint + if (multiEndpoints.containsKey(options.getName())) { + multiEndpoints.get(options.getName()).setEndpoints(options.getEndpoints()); + } else { + multiEndpoints.put(options.getName(), + (new MultiEndpoint.Builder(options.getEndpoints())) + .withRecoveryTimeout(options.getRecoveryTimeout()) + .build()); + } + }); + // Must have all multiendpoints before initializing the pools so that all multiendpoints + // can get status update of every pool. + meOptions.forEach(options -> { + // Create missing pools + options.getEndpoints().forEach(endpoint -> { + currentEndpoints.add(endpoint); + pools.computeIfAbsent(endpoint, e -> { + ManagedChannelBuilder managedChannelBuilder; + if (options.getChannelCredentials() != null) { + managedChannelBuilder = Grpc.newChannelBuilder(e, options.getChannelCredentials()); + } else { + String serviceAddress; + int port; + int colon = e.lastIndexOf(':'); + if (colon < 0) { + serviceAddress = e; + // Assume https by default. + port = 443; + } else { + serviceAddress = e.substring(0, colon); + port = Integer.parseInt(e.substring(colon + 1)); + } + managedChannelBuilder = ManagedChannelBuilder.forAddress(serviceAddress, port); + } + if (options.getChannelConfigurator() != null) { + managedChannelBuilder = options.getChannelConfigurator().apply(managedChannelBuilder); + } + + GcpManagedChannel channel = new GcpManagedChannel( + managedChannelBuilder, + apiConfig, + // Add endpoint to metric labels. + prepareGcpManagedChannelConfig(gcpManagedChannelOptions, e)); + // Start monitoring the pool state. + new EndpointStateMonitor(channel, e); + return channel; + }); + // Communicate current state to MultiEndpoints. + checkPoolState(pools.get(endpoint), endpoint); + }); + }); + defaultMultiEndpoint = multiEndpoints.get(meOptions.get(0).getName()); + + // Remove obsolete multiendpoints. + multiEndpoints.keySet().removeIf(name -> !currentMultiEndpoints.contains(name)); + + // Shutdown and remove the pools not present in options. + for (String endpoint : pools.keySet()) { + if (!currentEndpoints.contains(endpoint)) { + pools.get(endpoint).shutdown(); + pools.remove(endpoint); + } + } + } + + /** + * Initiates an orderly shutdown in which preexisting calls continue but new calls are immediately + * cancelled. + * + * @return this + * @since 1.0.0 + */ + @Override + public ManagedChannel shutdown() { + pools.values().forEach(GcpManagedChannel::shutdown); + return this; + } + + /** + * Returns whether the channel is shutdown. Shutdown channels immediately cancel any new calls, + * but may still have some calls being processed. + * + * @see #shutdown() + * @see #isTerminated() + * @since 1.0.0 + */ + @Override + public boolean isShutdown() { + return pools.values().stream().allMatch(GcpManagedChannel::isShutdown); + } + + /** + * Returns whether the channel is terminated. Terminated channels have no running calls and + * relevant resources released (like TCP connections). + * + * @see #isShutdown() + * @since 1.0.0 + */ + @Override + public boolean isTerminated() { + return pools.values().stream().allMatch(GcpManagedChannel::isTerminated); + } + + /** + * Initiates a forceful shutdown in which preexisting and new calls are cancelled. Although + * forceful, the shutdown process is still not instantaneous; {@link #isTerminated()} will likely + * return {@code false} immediately after this method returns. + * + * @return this + * @since 1.0.0 + */ + @Override + public ManagedChannel shutdownNow() { + pools.values().forEach(GcpManagedChannel::shutdownNow); + return this; + } + + /** + * Waits for the channel to become terminated, giving up if the timeout is reached. + * + * @return whether the channel is terminated, as would be done by {@link #isTerminated()}. + * @since 1.0.0 + */ + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + long endTimeNanos = System.nanoTime() + unit.toNanos(timeout); + for (GcpManagedChannel gcpManagedChannel : pools.values()) { + if (gcpManagedChannel.isTerminated()) { + continue; + } + long awaitTimeNanos = endTimeNanos - System.nanoTime(); + if (awaitTimeNanos <= 0) { + break; + } + gcpManagedChannel.awaitTermination(awaitTimeNanos, NANOSECONDS); + } + return isTerminated(); + } + + /** + * Check the value of {@link #ME_KEY} key in the {@link CallOptions} and if found use + * the MultiEndpoint with the same name for this call. + * + *

Create a {@link ClientCall} to the remote operation specified by the given {@link + * MethodDescriptor}. The returned {@link ClientCall} does not trigger any remote behavior until + * {@link ClientCall#start(Listener, Metadata)} is invoked. + * + * @param methodDescriptor describes the name and parameter types of the operation to call. + * @param callOptions runtime options to be applied to this call. + * @return a {@link ClientCall} bound to the specified method. + * @since 1.0.0 + */ + @Override + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + final String multiEndpointKey = callOptions.getOption(ME_KEY); + MultiEndpoint me = defaultMultiEndpoint; + if (multiEndpointKey != null) { + me = multiEndpoints.getOrDefault(multiEndpointKey, defaultMultiEndpoint); + } + return pools.get(me.getCurrentId()).newCall(methodDescriptor, callOptions); + } + + /** + * The authority of the destination this channel connects to. Typically this is in the format + * {@code host:port}. + * + * This may return different values over time because MultiEndpoint may switch between endpoints. + * + * @since 1.0.0 + */ + @Override + public String authority() { + return pools.get(defaultMultiEndpoint.getCurrentId()).authority(); + } +} diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java new file mode 100644 index 00000000..c1c30bac --- /dev/null +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointOptions.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.grpc; + +import com.google.api.core.ApiFunction; +import com.google.cloud.grpc.multiendpoint.MultiEndpoint; +import com.google.common.base.Preconditions; +import io.grpc.ChannelCredentials; +import io.grpc.ManagedChannelBuilder; +import java.time.Duration; +import java.util.List; + +/** + * {@link MultiEndpoint} configuration for the {@link GcpMultiEndpointChannel}. + */ +public class GcpMultiEndpointOptions { + + private final String name; + private final List endpoints; + private final ApiFunction, ManagedChannelBuilder> channelConfigurator; + private final ChannelCredentials channelCredentials; + private final Duration recoveryTimeout; + + public static String DEFAULT_NAME = "default"; + + public GcpMultiEndpointOptions(Builder builder) { + this.name = builder.name; + this.endpoints = builder.endpoints; + this.channelConfigurator = builder.channelConfigurator; + this.channelCredentials = builder.channelCredentials; + this.recoveryTimeout = builder.recoveryTimeout; + } + + /** + * Creates a new GcpMultiEndpointOptions.Builder. + * + * @param endpoints list of endpoints for the MultiEndpoint. + */ + public static Builder newBuilder(List endpoints) { + return new Builder(endpoints); + } + + /** + * Creates a new GcpMultiEndpointOptions.Builder from GcpMultiEndpointOptions. + */ + public static Builder newBuilder(GcpMultiEndpointOptions options) { + return new Builder(options); + } + + public String getName() { + return name; + } + + public List getEndpoints() { + return endpoints; + } + + public ApiFunction, ManagedChannelBuilder> getChannelConfigurator() { + return channelConfigurator; + } + + public ChannelCredentials getChannelCredentials() { + return channelCredentials; + } + + public Duration getRecoveryTimeout() { + return recoveryTimeout; + } + + public static class Builder { + + private String name = GcpMultiEndpointOptions.DEFAULT_NAME; + private List endpoints; + private ApiFunction, ManagedChannelBuilder> channelConfigurator; + private ChannelCredentials channelCredentials; + private Duration recoveryTimeout = Duration.ZERO; + + public Builder(List endpoints) { + setEndpoints(endpoints); + } + + public Builder(GcpMultiEndpointOptions options) { + this.name = options.getName(); + this.endpoints = options.getEndpoints(); + this.channelConfigurator = options.getChannelConfigurator(); + this.channelCredentials = options.getChannelCredentials(); + this.recoveryTimeout = options.getRecoveryTimeout(); + } + + public GcpMultiEndpointOptions build() { + return new GcpMultiEndpointOptions(this); + } + + private void setEndpoints(List endpoints) { + Preconditions.checkNotNull(endpoints); + Preconditions.checkArgument( + !endpoints.isEmpty(), "At least one endpoint must be specified."); + Preconditions.checkArgument( + endpoints.stream().noneMatch(s -> s.trim().isEmpty()), "No empty endpoints allowed."); + this.endpoints = endpoints; + } + + /** + * Sets the name of the MultiEndpoint. + * + * @param name MultiEndpoint name. + */ + public GcpMultiEndpointOptions.Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Sets the endpoints of the MultiEndpoint. + * + * @param endpoints List of endpoints in the form of host:port in descending priority order. + */ + public GcpMultiEndpointOptions.Builder withEndpoints(List endpoints) { + this.setEndpoints(endpoints); + return this; + } + + /** + * Sets the channel configurator for the MultiEndpoint channel pool. + * + * @param channelConfigurator function to perform on the ManagedChannelBuilder in the channel + * pool. + */ + public GcpMultiEndpointOptions.Builder withChannelConfigurator( + ApiFunction, ManagedChannelBuilder> channelConfigurator) { + this.channelConfigurator = channelConfigurator; + return this; + } + + /** + * Sets the channel credentials to use in the MultiEndpoint channel pool. + * + * @param channelCredentials channel credentials. + */ + public GcpMultiEndpointOptions.Builder withChannelCredentials( + ChannelCredentials channelCredentials) { + this.channelCredentials = channelCredentials; + return this; + } + + /** + * Sets the recovery timeout for the MultiEndpoint. See more info in the {@link MultiEndpoint}. + * + * @param recoveryTimeout recovery timeout. + */ + public GcpMultiEndpointOptions.Builder withRecoveryTimeout(Duration recoveryTimeout) { + this.recoveryTimeout = recoveryTimeout; + return this; + } + } +} diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java index 0d3d3414..56b890e7 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java @@ -16,12 +16,29 @@ package com.google.cloud.grpc; +import static com.google.cloud.grpc.GcpMultiEndpointChannel.ME_KEY; +import static com.google.cloud.spanner.SpannerOptions.CALL_CONTEXT_CONFIGURATOR_KEY; import static com.google.common.base.Preconditions.checkState; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import com.google.api.core.ApiFunction; +import com.google.api.gax.grpc.GrpcCallContext; +import com.google.api.gax.grpc.GrpcTransportChannel; import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.api.gax.rpc.TransportChannelProvider; import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.grpc.GcpManagedChannel.ChannelRef; +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions; +import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions; +import com.google.cloud.grpc.MetricRegistryTestUtils.FakeMetricRegistry; +import com.google.cloud.grpc.MetricRegistryTestUtils.MetricsRecord; +import com.google.cloud.grpc.MetricRegistryTestUtils.PointWithFunction; +import com.google.cloud.grpc.proto.ApiConfig; import com.google.cloud.spanner.Database; import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseClient; @@ -32,13 +49,17 @@ import com.google.cloud.spanner.InstanceConfigId; import com.google.cloud.spanner.InstanceId; import com.google.cloud.spanner.InstanceInfo; +import com.google.cloud.spanner.KeySet; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.SpannerOptions.CallContextConfigurator; import com.google.common.collect.Iterators; import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.Empty; +import com.google.protobuf.util.JsonFormat; +import com.google.protobuf.util.JsonFormat.Parser; import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import com.google.spanner.admin.instance.v1.CreateInstanceMetadata; import com.google.spanner.v1.BatchCreateSessionsRequest; @@ -62,14 +83,24 @@ import com.google.spanner.v1.SpannerGrpc.SpannerFutureStub; import com.google.spanner.v1.SpannerGrpc.SpannerStub; import com.google.spanner.v1.TransactionOptions; +import com.google.spanner.v1.TransactionOptions.ReadOnly; +import com.google.spanner.v1.TransactionOptions.ReadWrite; import com.google.spanner.v1.TransactionSelector; +import io.grpc.CallOptions; import io.grpc.ConnectivityState; +import io.grpc.Context; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.MethodDescriptor; import io.grpc.StatusRuntimeException; import io.grpc.auth.MoreCallCredentials; import io.grpc.stub.StreamObserver; +import io.opencensus.metrics.LabelValue; import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -77,13 +108,14 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.function.Function; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -114,7 +146,7 @@ public final class SpannerIntegrationTest { private static final int MAX_CHANNEL = 3; private static final int MAX_STREAM = 2; - private static final ManagedChannelBuilder builder = + private static final ManagedChannelBuilder builder = ManagedChannelBuilder.forAddress(SPANNER_TARGET, 443); private GcpManagedChannel gcpChannel; private GcpManagedChannel gcpChannelBRR; @@ -149,7 +181,7 @@ private Level lastLogLevel() { private final Handler testLogHandler = new Handler() { @Override - public void publish(LogRecord record) { + public synchronized void publish(LogRecord record) { logRecords.add(record); } @@ -259,7 +291,7 @@ private SpannerBlockingStub getSpannerBlockingStub() { return stub; } - private static void deleteSession(SpannerGrpc.SpannerBlockingStub stub, Session session) { + private static void deleteSession(SpannerBlockingStub stub, Session session) { if (session != null) { stub.deleteSession(DeleteSessionRequest.newBuilder().setName(session.getName()).build()); } @@ -316,7 +348,7 @@ private List createAsyncSessions(SpannerStub stub) throws Exception { // Check CreateSession with multiple channels and streams, CreateSessionRequest req = CreateSessionRequest.newBuilder().setDatabase(DATABASE_PATH).build(); for (int i = 0; i < MAX_CHANNEL * MAX_STREAM; i++) { - AsyncResponseObserver resp = new AsyncResponseObserver(); + AsyncResponseObserver resp = new AsyncResponseObserver<>(); stub.createSession(req, resp); resps.add(resp); } @@ -337,7 +369,7 @@ private void deleteAsyncSessions(SpannerStub stub, List respNames) throw AsyncResponseObserver resp = new AsyncResponseObserver<>(); stub.deleteSession(DeleteSessionRequest.newBuilder().setName(respName).build(), resp); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(respName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -387,7 +419,7 @@ private void deleteFutureSessions(SpannerFutureStub stub, List futureNam ListenableFuture future = stub.deleteSession(DeleteSessionRequest.newBuilder().setName(futureName).build()); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(futureName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -416,7 +448,7 @@ public void setupChannels() { .withApiConfigJsonFile(configFile) .withOptions(GcpManagedChannelOptions.newBuilder() .withChannelPoolOptions( - GcpManagedChannelOptions.GcpChannelPoolOptions.newBuilder() + GcpChannelPoolOptions.newBuilder() .setMaxSize(MAX_CHANNEL) .setConcurrentStreamsLowWatermark(MAX_STREAM) .setUseRoundRobinOnBind(true) @@ -429,12 +461,260 @@ public void setupChannels() { public void shutdownChannels() { testLogger.removeHandler(testLogHandler); testLogger.setLevel(Level.INFO); + logRecords.clear(); gcpChannel.shutdownNow(); gcpChannelBRR.shutdownNow(); } + private long getOkCallsCount( + FakeMetricRegistry fakeRegistry, String endpoint) { + MetricsRecord record = fakeRegistry.pollRecord(); + List> metric = + record.getMetrics().get(GcpMetricsConstants.METRIC_NUM_CALLS_COMPLETED); + for (PointWithFunction m : metric) { + assertThat(m.keys().get(0).getKey()).isEqualTo("result"); + assertThat(m.keys().get(1).getKey()).isEqualTo("endpoint"); + if (!m.values().get(0).equals(LabelValue.create(GcpMetricsConstants.RESULT_SUCCESS))) { + continue; + } + if (!m.values().get(1).equals(LabelValue.create(endpoint))) { + continue; + } + return m.value(); + } + fail("Success calls metric is not found for endpoint: " + endpoint); + return 0; + } + + // For this test we'll create a Spanner client with gRPC-GCP MultiEndpoint feature. + // + // Imagine we have a multi-region Spanner instance with leader in the us-east4 and follower in the + // us-east1 regions. + // + // We will provide two MultiEndpoint configs: "leader" (having leader region endpoint first and + // follower second) and "follower" (having follower region endpoint first and leader second). + // + // Then we'll make sure the Spanner client uses leader MultiEndpoint as a default one and creates + // its sessions there. Then we'll make sure a read request will also use the leader MultiEndpoint + // by default. + // + // Then we'll verify we can use the follower MultiEndpoint when needed by specifying that in + // the Spanner context. + // + // Then we'll update MultiEndpoints configuration by replacing the leader endpoint and renaming + // the follower MultiEndpoint. And make sure the new leader endpoint and the previous follower + // endpoint are still working as expected when using different MultiEndpoints. + @Test + public void testSpannerMultiEndpointClient() throws IOException, InterruptedException { + // Watch debug messages. + testLogger.setLevel(Level.FINEST); + + final FakeMetricRegistry fakeRegistry = new FakeMetricRegistry(); + + File configFile = + new File(SpannerIntegrationTest.class.getClassLoader().getResource(API_FILE).getFile()); + + // Leader-first multi-endpoint endpoints. + final List leaderEndpoints = new ArrayList<>(); + // Follower-first multi-endpoint endpoints. + final List followerEndpoints = new ArrayList<>(); + final String leaderEndpoint = "us-east4.googleapis.com:443"; + final String followerEndpoint = "us-east1.googleapis.com:443"; + leaderEndpoints.add(leaderEndpoint); + leaderEndpoints.add(followerEndpoint); + followerEndpoints.add(followerEndpoint); + followerEndpoints.add(leaderEndpoint); + + ApiFunction, ManagedChannelBuilder> configurator = input -> input.overrideAuthority( + SPANNER_TARGET); + + GcpMultiEndpointOptions leaderOpts = GcpMultiEndpointOptions.newBuilder(leaderEndpoints) + .withName("leader") + .withChannelConfigurator(configurator) + .withRecoveryTimeout(Duration.ofSeconds(3)) + .build(); + + GcpMultiEndpointOptions followerOpts = GcpMultiEndpointOptions.newBuilder(followerEndpoints) + .withName("follower") + .withChannelConfigurator(configurator) + .withRecoveryTimeout(Duration.ofSeconds(3)) + .build(); + + List opts = new ArrayList<>(); + opts.add(leaderOpts); + opts.add(followerOpts); + + Parser parser = JsonFormat.parser(); + ApiConfig.Builder apiConfig = ApiConfig.newBuilder(); + Reader reader = Files.newBufferedReader(configFile.toPath(), UTF_8); + parser.merge(reader, apiConfig); + + GcpMultiEndpointChannel gcpMultiEndpointChannel = new GcpMultiEndpointChannel( + opts, + apiConfig.build(), + GcpManagedChannelOptions.newBuilder() + .withChannelPoolOptions(GcpChannelPoolOptions.newBuilder() + .setConcurrentStreamsLowWatermark(0) + .setMaxSize(3) + .build()) + .withMetricsOptions(GcpMetricsOptions.newBuilder() + .withMetricRegistry(fakeRegistry) + .build()) + .build()); + + final int currentIndex = GcpManagedChannel.channelPoolIndex.get(); + final String followerPoolIndex = String.format("pool-%d", currentIndex); + final String leaderPoolIndex = String.format("pool-%d", currentIndex - 1); + + TimeUnit.MILLISECONDS.sleep(200); + + List logMessages = logRecords.stream() + .map(LogRecord::getMessage).collect(Collectors.toList()); + + // Make sure min channels are created and connections are established right away in both pools. + for (String poolIndex : Arrays.asList(leaderPoolIndex, followerPoolIndex)) { + for (int i = 0; i < 2; i++) { + assertThat(logMessages).contains( + poolIndex + ": Channel " + i + " state change detected: null -> IDLE"); + + assertThat(logMessages).contains( + poolIndex + ": Channel " + i + " state change detected: IDLE -> CONNECTING"); + } + } + + // Make sure endpoint is set as a metric label for each pool. + assertThat(logRecords.stream().filter(logRecord -> + logRecord.getMessage().matches( + leaderPoolIndex + ": Metrics options: \\{namePrefix: \"\", labels: \\[endpoint: " + + "\"" + leaderEndpoint + "\"], metricRegistry: .*" + )).count()).isEqualTo(1); + + assertThat(logRecords.stream().filter(logRecord -> + logRecord.getMessage().matches( + followerPoolIndex + ": Metrics options: \\{namePrefix: \"\", labels: \\[endpoint: " + + "\"" + followerEndpoint + "\"], metricRegistry: .*" + )).count()).isEqualTo(1); + + logRecords.clear(); + + TransportChannelProvider channelProvider = FixedTransportChannelProvider.create( + GrpcTransportChannel.create(gcpMultiEndpointChannel)); + + SpannerOptions.Builder options = SpannerOptions.newBuilder().setProjectId(GCP_PROJECT_ID); + + options.setChannelProvider(channelProvider); + // Match channel pool size. + options.setNumChannels(3); + + Spanner spanner = options.build().getService(); + InstanceId instanceId = InstanceId.of(GCP_PROJECT_ID, INSTANCE_ID); + DatabaseId databaseId = DatabaseId.of(instanceId, DB_NAME); + DatabaseClient databaseClient = spanner.getDatabaseClient(databaseId); + + Runnable readQuery = () -> { + try (com.google.cloud.spanner.ResultSet read = databaseClient.singleUse() + .read("Users", KeySet.all(), Arrays.asList("UserId", "UserName"))) { + int readRows = 0; + while (read.next()) { + readRows++; + assertEquals(USERNAME, read.getCurrentRowAsStruct().getString("UserName")); + } + assertEquals(1, readRows); + } + }; + + // Make sure leader endpoint is used by default. + assertThat(getOkCallsCount(fakeRegistry, leaderEndpoint)).isEqualTo(0); + readQuery.run(); + + // Wait for sessions creation requests to be completed but no more than 10 seconds. + for (int i = 0; i < 20; i++) { + TimeUnit.MILLISECONDS.sleep(500); + if (getOkCallsCount(fakeRegistry, leaderEndpoint) == 4) { + break; + } + } + + // 3 session creation requests + 1 our read request to the leader endpoint. + assertThat(getOkCallsCount(fakeRegistry, leaderEndpoint)).isEqualTo(4); + + // Make sure there were 3 session creation requests in the leader pool only. + assertThat(logRecords.stream().filter(logRecord -> + logRecord.getMessage().matches( + leaderPoolIndex + ": Binding \\d+ key\\(s\\) to channel \\d:.*" + )).count()).isEqualTo(3); + + assertThat(logRecords.stream().filter(logRecord -> + logRecord.getMessage().matches( + followerPoolIndex + ": Binding \\d+ key\\(s\\) to channel \\d:.*" + )).count()).isEqualTo(0); + + // Create context for using follower-first multi-endpoint. + Function contextFor = meName -> Context.current() + .withValue(CALL_CONTEXT_CONFIGURATOR_KEY, + new CallContextConfigurator() { + @Nullable + @Override + public ApiCallContext configure(ApiCallContext context, ReqT request, + MethodDescriptor method) { + return context.merge(GrpcCallContext.createDefault().withCallOptions( + CallOptions.DEFAULT.withOption(ME_KEY, meName))); + } + }); + + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(0); + // Use follower, make sure it is used. + contextFor.apply("follower").run(readQuery); + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(1); + + // Replace leader endpoints. + final String newLeaderEndpoint = "us-west1.googleapis.com:443"; + leaderEndpoints.clear(); + leaderEndpoints.add(newLeaderEndpoint); + leaderEndpoints.add(followerEndpoint); + leaderOpts = GcpMultiEndpointOptions.newBuilder(leaderEndpoints) + .withName("leader") + .withChannelConfigurator(configurator) + .build(); + + followerEndpoints.clear(); + followerEndpoints.add(followerEndpoint); + followerEndpoints.add(newLeaderEndpoint); + + // Rename follower MultiEndpoint. + followerOpts = GcpMultiEndpointOptions.newBuilder(followerEndpoints) + .withName("follower-2") + .withChannelConfigurator(configurator) + .build(); + + opts.clear(); + opts.add(leaderOpts); + opts.add(followerOpts); + + gcpMultiEndpointChannel.setMultiEndpoints(opts); + + // As it takes some time to connect to the new leader endpoint, RPC will fall back to the + // follower until we connect to leader. + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(1); + readQuery.run(); + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(2); + + TimeUnit.MILLISECONDS.sleep(500); + + // Make sure the new leader endpoint is used by default after it is connected. + assertThat(getOkCallsCount(fakeRegistry, newLeaderEndpoint)).isEqualTo(0); + readQuery.run(); + assertThat(getOkCallsCount(fakeRegistry, newLeaderEndpoint)).isEqualTo(1); + + // Make sure that the follower endpoint still works if specified. + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(2); + // Use follower, make sure it is used. + contextFor.apply("follower-2").run(readQuery); + assertThat(getOkCallsCount(fakeRegistry, followerEndpoint)).isEqualTo(3); + } + @Test - public void testCreateAndGetSessionBlocking() throws Exception { + public void testCreateAndGetSessionBlocking() { SpannerBlockingStub stub = getSpannerBlockingStub(); CreateSessionRequest req = CreateSessionRequest.newBuilder().setDatabase(DATABASE_PATH).build(); // The first MAX_CHANNEL requests (without affinity) should be distributed 1 per channel. @@ -457,7 +737,7 @@ public void testCreateAndGetSessionBlocking() throws Exception { } @Test - public void testBatchCreateSessionsBlocking() throws Exception { + public void testBatchCreateSessionsBlocking() { int sessionCount = 10; SpannerBlockingStub stub = getSpannerBlockingStub(); BatchCreateSessionsRequest req = @@ -513,7 +793,7 @@ public void testSessionsCreatedUsingRoundRobin() throws Exception { .setSql("select * FROM Users") .build()); // The ChannelRef which is bound with the lastSession. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannelBRR.affinityKeyToChannelRef.get(lastSession); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -581,7 +861,7 @@ public void testSessionsCreatedWithoutRoundRobin() throws Exception { .setSql("select * FROM Users") .build()); // The ChannelRef which is bound with the lastSession. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(lastSession); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -638,7 +918,7 @@ public void testExecuteSqlFuture() throws Exception { .setSql("select * FROM Users") .build()); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(futureName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -660,7 +940,7 @@ public void testExecuteStreamingSqlAsync() throws Exception { ExecuteSqlRequest.newBuilder().setSession(respName).setSql("select * FROM Users").build(), resp); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(respName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -678,7 +958,7 @@ public void testPartitionQueryAsync() throws Exception { for (String respName : respNames) { TransactionOptions options = TransactionOptions.newBuilder() - .setReadOnly(TransactionOptions.ReadOnly.getDefaultInstance()) + .setReadOnly(ReadOnly.getDefaultInstance()) .build(); TransactionSelector selector = TransactionSelector.newBuilder().setBegin(options).build(); AsyncResponseObserver resp = new AsyncResponseObserver<>(); @@ -690,7 +970,7 @@ public void testPartitionQueryAsync() throws Exception { .build(), resp); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(respName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -707,7 +987,7 @@ public void testExecuteBatchDmlFuture() throws Exception { for (String futureName : futureNames) { TransactionOptions options = TransactionOptions.newBuilder() - .setReadWrite(TransactionOptions.ReadWrite.getDefaultInstance()) + .setReadWrite(ReadWrite.getDefaultInstance()) .build(); TransactionSelector selector = TransactionSelector.newBuilder().setBegin(options).build(); // Will use only one session for the whole batch. @@ -719,7 +999,7 @@ public void testExecuteBatchDmlFuture() throws Exception { .addStatements(Statement.newBuilder().setSql("select * FROM Users").build()) .build()); // The ChannelRef which is bound with the current affinity key. - GcpManagedChannel.ChannelRef currentChannel = + ChannelRef currentChannel = gcpChannel.affinityKeyToChannelRef.get(futureName); // Verify the channel is in use. assertEquals(1, currentChannel.getActiveStreamsCount()); @@ -769,7 +1049,7 @@ private static class AsyncResponseObserver implements StreamObserver Date: Wed, 6 Jul 2022 10:23:39 -0700 Subject: [PATCH 80/87] Address PR comments --- .../cloud/grpc/GcpMultiEndpointChannel.java | 16 +++++++++------- .../cloud/grpc/multiendpoint/Endpoint.java | 2 +- .../cloud/grpc/multiendpoint/MultiEndpoint.java | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java index a1b1276e..7ad57c7e 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java @@ -68,6 +68,15 @@ * GcpMultiEndpointOptions list defines the default MultiEndpoint that will be used when no * MultiEndpoint name is provided with an RPC call. * + *

Example: + * + *

Let's assume we have a service with read and write operations and the following backends: + *

    + *
  • service.example.com -- the main set of backends supporting all operations
    • + *
  • service-fallback.example.com -- read-write replica supporting all operations
    • + *
  • ro-service.example.com -- read-only replica supporting only read operations
    • + *
+ * *

Example configuration: *

    *
  • @@ -87,13 +96,6 @@ *
    • *
* - *

Let's assume we have a service with read and write operations and the following backends: - *

    - *
  • service.example.com -- the main set of backends supporting all operations
    • - *
  • service-fallback.example.com -- read-write replica supporting all operations
    • - *
  • ro-service.example.com -- read-only replica supporting only read operations
    • - *
- * *

With the configuration above GcpMultiEndpointChannel will use the "default" MultiEndpoint by * default. It means that RPC calls by default will use the main endpoint and if it is not available * then the read-write replica. diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java index d6f4debd..fc6d08eb 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/Endpoint.java @@ -67,7 +67,7 @@ public void setPriority(int priority) { public synchronized void setChangeStateFuture(ScheduledFuture future) { resetStateChangeFuture(); - this.changeStateFuture = future; + changeStateFuture = future; } public synchronized void resetStateChangeFuture() { diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java index c98f31d1..785b2477 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java @@ -37,7 +37,7 @@ * priority). MultiEndpoint returns top priority endpoint that is available as current. If no * endpoint is available, MultiEndpoint returns the top priority endpoint. * - *

Sometimes switching between endpoints can be costly, and it is worth to wait for some time + *

Sometimes switching between endpoints can be costly, and it is worth waiting for some time * after current endpoint becomes unavailable. For this case, use {@link * Builder#withRecoveryTimeout} to set the recovery timeout. MultiEndpoint will keep the current * endpoint for up to recovery timeout after it became unavailable to give it some time to recover. From 3884c96a11dddfe3230a8cfa81df45dff85a669a Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Wed, 6 Jul 2022 10:39:27 -0700 Subject: [PATCH 81/87] Address PR comments --- .../cloud/grpc/multiendpoint/MultiEndpoint.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java index 785b2477..c706e119 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java @@ -57,14 +57,11 @@ public final class MultiEndpoint { private String currentId; private final Duration recoveryTimeout; - private final boolean recoveryEnabled; private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private MultiEndpoint(Builder builder) { this.recoveryTimeout = builder.recoveryTimeout; - this.recoveryEnabled = - !builder.recoveryTimeout.isNegative() && !builder.recoveryTimeout.isZero(); this.setEndpoints(builder.endpoints); } @@ -113,6 +110,10 @@ private synchronized void setEndpointStateInternal(String endpointId, EndpointSt } } + private boolean isRecoveryEnabled() { + return !recoveryTimeout.isNegative() && !recoveryTimeout.isZero(); + } + /** Inform MultiEndpoint when an endpoint becomes available or unavailable. */ public synchronized void setEndpointAvailable(String endpointId, boolean available) { setEndpointState(endpointId, available ? EndpointState.AVAILABLE : EndpointState.UNAVAILABLE); @@ -125,7 +126,7 @@ private synchronized void setEndpointState(String endpointId, EndpointState stat return; } // If we allow some recovery time. - if (EndpointState.UNAVAILABLE.equals(state) && recoveryEnabled) { + if (EndpointState.UNAVAILABLE.equals(state) && isRecoveryEnabled()) { endpoint.setState(EndpointState.RECOVERING); ScheduledFuture future = executor.schedule( @@ -162,9 +163,9 @@ public synchronized void setEndpoints(List endpoints) { continue; } EndpointState newState = - recoveryEnabled ? EndpointState.RECOVERING : EndpointState.UNAVAILABLE; + isRecoveryEnabled() ? EndpointState.RECOVERING : EndpointState.UNAVAILABLE; Endpoint newEndpoint = new Endpoint(endpointId, newState, priority++); - if (recoveryEnabled) { + if (isRecoveryEnabled()) { ScheduledFuture future = executor.schedule( () -> setEndpointStateInternal(endpointId, EndpointState.UNAVAILABLE), From 285e76406a0c5795e18884d5ac1ee5310de804f3 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 6 Jul 2022 11:18:53 -0700 Subject: [PATCH 82/87] Upgrade the dependencies --- end2end-test-examples/gcs/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/end2end-test-examples/gcs/build.gradle b/end2end-test-examples/gcs/build.gradle index ebc0d149..0d8c1ef0 100644 --- a/end2end-test-examples/gcs/build.gradle +++ b/end2end-test-examples/gcs/build.gradle @@ -9,8 +9,8 @@ version '1.0-SNAPSHOT' sourceCompatibility = 1.8 -def gcsioVersion = '2.2.6' -def grpcVersion = '1.46.0' +def gcsioVersion = '2.2.7' +def grpcVersion = '1.47.0' def protobufVersion = '3.19.2' def protocVersion = protobufVersion def conscryptVersion = '2.5.2' From 282c0d776f4cea439dcfcee238c85d1c273e38af Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Thu, 7 Jul 2022 12:17:18 -0700 Subject: [PATCH 83/87] Add TODOs for endpoints credentials. --- .../com/google/cloud/grpc/GcpMultiEndpointChannel.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java index 7ad57c7e..90edfa8c 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java @@ -224,6 +224,9 @@ public void setMultiEndpoints(List meOptions) { Preconditions.checkArgument(!meOptions.isEmpty(), "MultiEndpoints list is empty"); Set currentMultiEndpoints = new HashSet<>(); Set currentEndpoints = new HashSet<>(); + + // Must have all multiendpoints before initializing the pools so that all multiendpoints + // can get status update of every pool. meOptions.forEach(options -> { currentMultiEndpoints.add(options.getName()); // Create or update MultiEndpoint @@ -236,8 +239,9 @@ public void setMultiEndpoints(List meOptions) { .build()); } }); - // Must have all multiendpoints before initializing the pools so that all multiendpoints - // can get status update of every pool. + + // TODO: Add support for different credentials for the same endpoint. + // TODO: Add support for DirectPath. meOptions.forEach(options -> { // Create missing pools options.getEndpoints().forEach(endpoint -> { From af5be1e5641ce86cc47f88e7d1f824ec550afa21 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Fri, 8 Jul 2022 11:14:22 -0700 Subject: [PATCH 84/87] Rename maybeFallback to maybeUpdateCurrentEndpoint. --- .../google/cloud/grpc/multiendpoint/MultiEndpoint.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java index c706e119..18b9abfd 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/multiendpoint/MultiEndpoint.java @@ -106,7 +106,7 @@ private synchronized void setEndpointStateInternal(String endpointId, EndpointSt Endpoint endpoint = endpointsMap.get(endpointId); if (endpoint != null) { endpoint.setState(state); - maybeFallback(); + maybeUpdateCurrentEndpoint(); } } @@ -138,7 +138,7 @@ private synchronized void setEndpointState(String endpointId, EndpointState stat } endpoint.resetStateChangeFuture(); endpoint.setState(state); - maybeFallback(); + maybeUpdateCurrentEndpoint(); } /** @@ -176,10 +176,12 @@ public synchronized void setEndpoints(List endpoints) { endpointsMap.put(endpointId, newEndpoint); } - maybeFallback(); + maybeUpdateCurrentEndpoint(); } - private synchronized void maybeFallback() { + // Updates currentId to the top-priority available endpoint unless the current endpoint is + // recovering. + private synchronized void maybeUpdateCurrentEndpoint() { Optional topEndpoint = endpointsMap.values().stream() .filter((c) -> c.getState().equals(EndpointState.AVAILABLE)) From 8080ea2e835dbaa9cbbf021c4bde9ec6d31c3943 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Fri, 8 Jul 2022 11:46:11 -0700 Subject: [PATCH 85/87] Add authorityFor method --- .../cloud/grpc/GcpMultiEndpointChannel.java | 21 +++++++++++++++++-- .../cloud/grpc/SpannerIntegrationTest.java | 8 +++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java index 90edfa8c..44e04178 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java @@ -394,8 +394,11 @@ public ClientCall newCall( } /** - * The authority of the destination this channel connects to. Typically this is in the format - * {@code host:port}. + * The authority of the current endpoint of the default MultiEndpoint. Typically, this is in the + * format {@code host:port}. + * + * To get the authority of the current endpoint of another MultiEndpoint use {@link + * #authorityFor(String)} method. * * This may return different values over time because MultiEndpoint may switch between endpoints. * @@ -405,4 +408,18 @@ public ClientCall newCall( public String authority() { return pools.get(defaultMultiEndpoint.getCurrentId()).authority(); } + + /** + * The authority of the current endpoint of the specified MultiEndpoint. Typically, this is in the + * format {@code host:port}. + * + * This may return different values over time because MultiEndpoint may switch between endpoints. + */ + public String authorityFor(String multiEndpointName) { + MultiEndpoint multiEndpoint = multiEndpoints.get(multiEndpointName); + if (multiEndpoint == null) { + return null; + } + return pools.get(multiEndpoint.getCurrentId()).authority(); + } } diff --git a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java index 56b890e7..780c6f58 100644 --- a/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java +++ b/grpc-gcp/src/test/java/com/google/cloud/grpc/SpannerIntegrationTest.java @@ -566,6 +566,14 @@ public void testSpannerMultiEndpointClient() throws IOException, InterruptedExce final String followerPoolIndex = String.format("pool-%d", currentIndex); final String leaderPoolIndex = String.format("pool-%d", currentIndex - 1); + // Make sure authorities are overridden by channel configurator. + assertThat(gcpMultiEndpointChannel.authority()).isEqualTo(SPANNER_TARGET); + assertThat(gcpMultiEndpointChannel.authorityFor("leader")) + .isEqualTo(SPANNER_TARGET); + assertThat(gcpMultiEndpointChannel.authorityFor("follower")) + .isEqualTo(SPANNER_TARGET); + assertThat(gcpMultiEndpointChannel.authorityFor("no-such-name")).isNull(); + TimeUnit.MILLISECONDS.sleep(200); List logMessages = logRecords.stream() From 34e77dadef6a4d4754cc2bfc64761c56d7c51812 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Fri, 8 Jul 2022 11:48:30 -0700 Subject: [PATCH 86/87] Update TODOs for different channel credentials. --- .../java/com/google/cloud/grpc/GcpMultiEndpointChannel.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java index 44e04178..9d9de7e7 100644 --- a/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java +++ b/grpc-gcp/src/main/java/com/google/cloud/grpc/GcpMultiEndpointChannel.java @@ -240,8 +240,10 @@ public void setMultiEndpoints(List meOptions) { } }); - // TODO: Add support for different credentials for the same endpoint. - // TODO: Add support for DirectPath. + // TODO: Support the same endpoint in different MultiEndpoint to use different channel + // credentials. + // TODO: Support different endpoints in the same MultiEndpoint to use different channel + // credentials. meOptions.forEach(options -> { // Create missing pools options.getEndpoints().forEach(endpoint -> { From a6826b29835c2eed77ce657558e507e082777173 Mon Sep 17 00:00:00 2001 From: Yuri Golobokov Date: Sun, 17 Jul 2022 19:26:04 -0700 Subject: [PATCH 87/87] New release 1.2.0 --- grpc-gcp/CHANGELOG.md | 13 +++++++++++++ grpc-gcp/build.gradle | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/grpc-gcp/CHANGELOG.md b/grpc-gcp/CHANGELOG.md index 20597001..94d17bc7 100644 --- a/grpc-gcp/CHANGELOG.md +++ b/grpc-gcp/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 1.2.0 (2021-07-18) + +### Features + +* multi-endpoint (#135). +* round-robin for bind calls (#127). +* minSize of the channel pool (#134). +* detailed logs (#133). +* log metrics (#131). +* map a key to current channel when bound call arrives but no mapping for the +key exists (#132). +* consolidate channel pool config in the GcpChannelPoolOptions (#109). + ## 1.1.0 (2021-07-20) ### Features diff --git a/grpc-gcp/build.gradle b/grpc-gcp/build.gradle index 32e37965..63ab6499 100644 --- a/grpc-gcp/build.gradle +++ b/grpc-gcp/build.gradle @@ -15,7 +15,7 @@ repositories { mavenLocal() } -version = '1.1.1-SNAPSHOT' +version = '1.2.0' group = 'com.google.cloud' description = 'GRPC-GCP-Extension Java' sourceCompatibility = '1.8'