googleapis · chingor13 · Sep 6, 2019 · Aug 14, 2019
diff --git a/google-cloud-testing/storage-benchwrapper/README.md b/google-cloud-testing/storage-benchwrapper/README.md
@@ -0,0 +1,54 @@
+# storage-benchwrapper
+
+storage-benchwrapper is a gRPC wrapper around the storage library for benchmarking purposes.
+
+## Running
+
+```
+mvn clean install -DskipTests=true (one time)
+cd google-cloud-testing/storage-benchwrapper
+export STORAGE_EMULATOR_HOST=localhost:8080
+mvn clean install exec:java -DskipTests=true -Dport=8081
+```
+
+## Generating .proto sources
+
+Sources are generated as part of the protobuf-maven-plugin plugin, but if you'd
+like to generate them yourself to see the output you can run:
+
+```
+cd google-cloud-testing/storage-benchwrapper
+protoc \
+    --plugin=protoc-gen-grpc-java=build/exe/java_plugin/protoc-gen-grpc-java \
+    --java_out=src/main/java \
+    --grpc-java_out=src/main/java \
+    --proto_path=src/main/proto \
+    src/main/proto/*.proto
+```
+
+Note that you'll need to delete these, or uncomment the plugin, since multiple
+definitions of the same class are not allowed.
+
+Note that these should not be committed into git.
+
+Note that you can also `mvn compile -DskipTests=true` and see sources in
+`target/generated-sources/`.
+
+## Debugging HTTP requests
+
+To debug HTTP requests, place a `logging.properties` at
+`google-cloud-testing/storage-benchwrapper/logging.properties` with the
+following contents:
+
+```
+# Properties file which configures the operation of the JDK logging facility.
+# The system will look for this config file to be specified as a system property:
+# -Djava.util.logging.config.file=${project_loc:googleplus-simple-cmdline-sample}/logging.properties
+
+# Set up the console handler (uncomment "level" to show more fine-grained messages)
+handlers = java.util.logging.ConsoleHandler
+java.util.logging.ConsoleHandler.level = CONFIG
+
+# Set up logging of HTTP requests and responses (uncomment "level" to show)
+com.google.api.client.http.level = CONFIG
+```
diff --git a/google-cloud-testing/storage-benchwrapper/pom.xml b/google-cloud-testing/storage-benchwrapper/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>benchwrapper</artifactId>
+  <version>0.0.0</version>
+  <packaging>jar</packaging>
+  <groupId>com.google.cloud</groupId>
+
+  <properties>
+    <grpcVersion>1.23.0</grpcVersion>
+    <protobufVersion>3.9.1</protobufVersion>
+  </properties>
+
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.5.0.Final</version>
+      </extension>
+    </extensions>
+
+    <plugins>
+      <!-- Auto-generate classes from .protos -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.6.1</version>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:${protobufVersion}:exe:${os.detected.classifier}</protocArtifact>
+          <pluginId>grpc-java</pluginId>
+          <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}</pluginArtifact>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>compile-custom</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <!-- Protobuf runtime -->
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.6.0</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>com.google.cloud.benchwrapper.Main</mainClass>
+          <executable>maven</executable>
+          <skip>false</skip>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <!-- storage -->
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <version>1.85.0</version>
+    </dependency>
+
+    <!-- protobuf  -->
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${protobufVersion}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-protobuf</artifactId>
+      <version>${grpcVersion}</version>
+    </dependency>
+
+    <!-- grpc -->
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty-shaded</artifactId>
+      <version>${grpcVersion}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-stub</artifactId>
+      <version>${grpcVersion}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty</artifactId>
+      <version>${grpcVersion}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <version>2.0.25.Final</version>
+    </dependency>
+  </dependencies>
+</project>
diff --git a/...-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java b/...-cloud-testing/storage-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.benchwrapper;
+
+import java.util.Properties;
+import io.grpc.Server;
+import io.grpc.netty.NettyServerBuilder;
+
+class Main {
+  public static void main(String[] args) throws Exception {
+    Properties properties = new Properties(System.getProperties());
+    String port = properties.getProperty("port");
+    if (port == null || port.equals("")) {
+      System.err.println("Usage: mvn clean install exec:java -DskipTests=true -Dport=8081");
+      System.exit(1);
+    }
+
+    String storageEmulatorHost = System.getenv("STORAGE_EMULATOR_HOST");
+    if (storageEmulatorHost == null || storageEmulatorHost.equals("")) {
+      // We could use system properties here too, but every other language uses
+      // an environment variable called STORAGE_EMULATOR_HOST, so the
+      // consistency is nice to maintain.
+      System.err.println("Please set STORAGE_EMULATOR_HOST=localhost:8080");
+      System.exit(1);
+    }
+
+    System.out.println("Server starting up...");
+
+    int portInt = Integer.parseInt(port);
+    final Server server = NettyServerBuilder
+      .forPort(portInt)
+      .addService(new StorageBenchWrapperImpl(storageEmulatorHost))
+      .build()
+      .start();
+
+    System.out.println("Server starting up... done. Listening on " + port);
+    Runtime.getRuntime().addShutdownHook(new Thread() {
+      @Override
+      public void run() {
+        // Use stderr here since the logger may have been reset by its JVM shutdown hook.
+        System.err.println("Shutting down gRPC server since JVM is shutting down");
+        server.shutdown();
+      }
+    });
+
+    server.awaitTermination();
+  }
+}
+
diff --git a/...age-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java b/...age-benchwrapper/src/main/java/com/google/cloud/benchwrapper/StorageBenchWrapperImpl.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.benchwrapper;
+
+import io.grpc.stub.StreamObserver;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import com.google.cloud.benchwrapper.StorageBenchWrapperGrpc.StorageBenchWrapperImplBase;
+import com.google.cloud.ReadChannel;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.StorageOptions;
+
+class StorageBenchWrapperImpl extends StorageBenchWrapperImplBase {
+  private Storage client;
+
+  public StorageBenchWrapperImpl(String storageEmulatorHost) {
+    client = StorageOptions.newBuilder()
+        .setHost("http://" + storageEmulatorHost)
+        .build()
+        .getService();
+  }
+
+  public void write(ObjectWrite request, StreamObserver<EmptyResponse> responseObserver) {
+    System.out.println("write has been called");
+    EmptyResponse reply = EmptyResponse.newBuilder().build();
+    responseObserver.onNext(reply);
+    responseObserver.onCompleted();
+  }
+
+  public void read(ObjectRead request, StreamObserver<EmptyResponse> responseObserver) {
+    System.out.println("read has been called");
+
+    Blob blob = client.get(BlobId.of(request.getBucketName(), request.getObjectName()));
+
+    try (ReadChannel reader = blob.reader()) {
+      ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
+      while (reader.read(bytes) > 0) {
+        bytes.flip();
+        // do nothing with bytes
+        bytes.clear();
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      System.exit(1);
+    }
+
+    EmptyResponse reply = EmptyResponse.newBuilder().build();
+    responseObserver.onNext(reply);
+    responseObserver.onCompleted();
+  }
+}
diff --git a/google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto b/google-cloud-testing/storage-benchwrapper/src/main/proto/storage.proto
@@ -0,0 +1,45 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package storage_bench;
+option java_multiple_files = true;
+option java_package = "com.google.cloud.benchwrapper";
+
+message ObjectRead{
+  // The bucket string identifier.
+  string bucketName = 1;
+  // The object/blob string identifier.
+  string objectName = 2;
+}
+
+message ObjectWrite{
+   // The bucket string identifier.
+  string bucketName = 1;
+  // The object/blob string identifiers.
+  string objectName = 2;
+  // The string containing the upload file path.
+  string destination = 3;
+}
+
+message EmptyResponse{
+}
+
+service StorageBenchWrapper{
+  // Performs an upload from a specific object.
+  rpc Write(ObjectWrite) returns (EmptyResponse) {}
+  // Read a specific object.
+  rpc Read(ObjectRead) returns (EmptyResponse){}
+}