From db88471f2b84cd4bd16e9e389f7daaedbca31945 Mon Sep 17 00:00:00 2001 From: Jean de Klerk Date: Wed, 14 Aug 2019 14:41:40 -0600 Subject: [PATCH] storage: add benchwrapper This adds a small gRPC server around the google-cloud-storage client library, which allows g3 benchmarking code to prod at the library and record measurements without needing to be written in Java / be in the same repository. Since g3 requires TLS on all connections, the server is TLS-ed with a dummy cert. These certs aren't intended to be private: the kokoro environment is the only place this gets run. (never in prod, etc) --- .../storage-benchwrapper/README.md | 54 ++++++++ .../storage-benchwrapper/pom.xml | 116 ++++++++++++++++++ .../com/google/cloud/benchwrapper/Main.java | 63 ++++++++++ .../benchwrapper/StorageBenchWrapperImpl.java | 69 +++++++++++ .../src/main/proto/storage.proto | 45 +++++++ 5 files changed, 347 insertions(+) create mode 100644 google-cloud-testing/storage-benchwrapper/README.md create mode 100644 google-cloud-testing/storage-benchwrapper/pom.xml create mode 100644 google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java create mode 100644 google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java create mode 100644 google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto diff --git a/google-cloud-testing/storage-benchwrapper/README.md b/google-cloud-testing/storage-benchwrapper/README.md new file mode 100644 index 000000000000..cabbdfc38b1b --- /dev/null +++ b/google-cloud-testing/storage-benchwrapper/README.md @@ -0,0 +1,54 @@ +# storage-benchwrapper + +storage-benchwrapper is a gRPC wrapper around the storage library for benchmarking purposes. + +## Running + +``` +mvn clean install -DskipTests=true (one time) +cd google-cloud-testing/storage-benchwrapper +export STORAGE_EMULATOR_HOST=localhost:8080 +mvn clean install exec:java -DskipTests=true -Dport=8081 +``` + +## Generating .proto sources + +Sources are generated as part of the protobuf-maven-plugin plugin, but if you'd +like to generate them yourself to see the output you can run: + +``` +cd google-cloud-testing/storage-benchwrapper +protoc \ + --plugin=protoc-gen-grpc-java=build/exe/java_plugin/protoc-gen-grpc-java \ + --java_out=src/main/java \ + --grpc-java_out=src/main/java \ + --proto_path=src/main/proto \ + src/main/proto/*.proto +``` + +Note that you'll need to delete these, or uncomment the plugin, since multiple +definitions of the same class are not allowed. + +Note that these should not be committed into git. + +Note that you can also `mvn compile -DskipTests=true` and see sources in +`target/generated-sources/`. + +## Debugging HTTP requests + +To debug HTTP requests, place a `logging.properties` at +`google-cloud-testing/storage-benchwrapper/logging.properties` with the +following contents: + +``` +# Properties file which configures the operation of the JDK logging facility. +# The system will look for this config file to be specified as a system property: +# -Djava.util.logging.config.file=${project_loc:googleplus-simple-cmdline-sample}/logging.properties + +# Set up the console handler (uncomment "level" to show more fine-grained messages) +handlers = java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level = CONFIG + +# Set up logging of HTTP requests and responses (uncomment "level" to show) +com.google.api.client.http.level = CONFIG +``` \ No newline at end of file diff --git a/google-cloud-testing/storage-benchwrapper/pom.xml b/google-cloud-testing/storage-benchwrapper/pom.xml new file mode 100644 index 000000000000..00bfdeaedc23 --- /dev/null +++ b/google-cloud-testing/storage-benchwrapper/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + benchwrapper + 0.0.0 + jar + com.google.cloud + + + 1.23.0 + 3.9.1 + + + + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobufVersion}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier} + + + + + compile + compile-custom + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + + exec + + + + + com.google.cloud.benchwrapper.Main + maven + false + + + + + + + + + com.google.cloud + google-cloud-storage + 1.85.0 + + + + + com.google.protobuf + protobuf-java + ${protobufVersion} + + + io.grpc + grpc-protobuf + ${grpcVersion} + + + + + io.grpc + grpc-netty-shaded + ${grpcVersion} + + + io.grpc + grpc-stub + ${grpcVersion} + + + io.grpc + grpc-netty + ${grpcVersion} + + + io.netty + netty-tcnative-boringssl-static + 2.0.25.Final + + + \ No newline at end of file diff --git a/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java b/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java new file mode 100644 index 000000000000..3abf488e9278 --- /dev/null +++ b/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.benchwrapper; + +import java.util.Properties; +import io.grpc.Server; +import io.grpc.netty.NettyServerBuilder; + +class Main { + public static void main(String[] args) throws Exception { + Properties properties = new Properties(System.getProperties()); + String port = properties.getProperty("port"); + if (port == null || port.equals("")) { + System.err.println("Usage: mvn clean install exec:java -DskipTests=true -Dport=8081"); + System.exit(1); + } + + String storageEmulatorHost = System.getenv("STORAGE_EMULATOR_HOST"); + if (storageEmulatorHost == null || storageEmulatorHost.equals("")) { + // We could use system properties here too, but every other language uses + // an environment variable called STORAGE_EMULATOR_HOST, so the + // consistency is nice to maintain. + System.err.println("Please set STORAGE_EMULATOR_HOST=localhost:8080"); + System.exit(1); + } + + System.out.println("Server starting up..."); + + int portInt = Integer.parseInt(port); + final Server server = NettyServerBuilder + .forPort(portInt) + .addService(new StorageBenchWrapperImpl(storageEmulatorHost)) + .build() + .start(); + + System.out.println("Server starting up... done. Listening on " + port); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // Use stderr here since the logger may have been reset by its JVM shutdown hook. + System.err.println("Shutting down gRPC server since JVM is shutting down"); + server.shutdown(); + } + }); + + server.awaitTermination(); + } +} + diff --git a/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java b/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java new file mode 100644 index 000000000000..1d8cf41e7b9b --- /dev/null +++ b/google-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.benchwrapper; + +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.nio.ByteBuffer; + +import com.google.cloud.benchwrapper.StorageBenchWrapperGrpc.StorageBenchWrapperImplBase; +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; + +class StorageBenchWrapperImpl extends StorageBenchWrapperImplBase { + private Storage client; + + public StorageBenchWrapperImpl(String storageEmulatorHost) { + client = StorageOptions.newBuilder() + .setHost("http://" + storageEmulatorHost) + .build() + .getService(); + } + + public void write(ObjectWrite request, StreamObserver responseObserver) { + System.out.println("write has been called"); + EmptyResponse reply = EmptyResponse.newBuilder().build(); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } + + public void read(ObjectRead request, StreamObserver responseObserver) { + System.out.println("read has been called"); + + Blob blob = client.get(BlobId.of(request.getBucketName(), request.getObjectName())); + + try (ReadChannel reader = blob.reader()) { + ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); + while (reader.read(bytes) > 0) { + bytes.flip(); + // do nothing with bytes + bytes.clear(); + } + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + + EmptyResponse reply = EmptyResponse.newBuilder().build(); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } +} diff --git a/google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto b/google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto new file mode 100644 index 000000000000..1ec0737d870b --- /dev/null +++ b/google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto @@ -0,0 +1,45 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package storage_bench; +option java_multiple_files = true; +option java_package = "com.google.cloud.benchwrapper"; + +message ObjectRead{ + // The bucket string identifier. + string bucketName = 1; + // The object/blob string identifier. + string objectName = 2; +} + +message ObjectWrite{ + // The bucket string identifier. + string bucketName = 1; + // The object/blob string identifiers. + string objectName = 2; + // The string containing the upload file path. + string destination = 3; +} + +message EmptyResponse{ +} + +service StorageBenchWrapper{ + // Performs an upload from a specific object. + rpc Write(ObjectWrite) returns (EmptyResponse) {} + // Read a specific object. + rpc Read(ObjectRead) returns (EmptyResponse){} +}