Merge remote-tracking branch 'upstream/master' into profiling

Signed-off-by: Otto van der Schaaf <[email protected]>
istio · oschaaf · Apr 15, 2020 · Apr 16, 2020 · Apr 16, 2020 · Apr 16, 2020
commit a3f5587b9ccdd654c44339f97b4dbabae7960038
@@ -22,7 +22,8 @@
 import shlex
 import uuid
 import sys
-
+import tempfile
+import time
 from subprocess import getoutput
 from urllib.parse import urlparse
 from threading import Thread
@@ -177,16 +178,16 @@ def compute_uri(self, svc, port_type):
             sys.exit("invalid load generator %s, must be fortio or nighthawk", self.load_gen_type)
 
     def nosidecar(self, load_gen_cmd, sidecar_mode):
-        return load_gen_cmd + "_" + sidecar_mode + " " + self.compute_uri(self.server.ip, "direct_port")
+        return load_gen_cmd + sidecar_mode + " " + self.compute_uri(self.server.ip, "direct_port")
 
     def serversidecar(self, load_gen_cmd, sidecar_mode):
-        return load_gen_cmd + "_" + sidecar_mode + " " + self.compute_uri(self.server.ip, "port")
+        return load_gen_cmd + sidecar_mode + " " + self.compute_uri(self.server.ip, "port")
 
     def clientsidecar(self, load_gen_cmd, sidecar_mode):
-        return load_gen_cmd + "_" + sidecar_mode + " " + self.compute_uri(self.server.labels["app"], "direct_port")
+        return load_gen_cmd + sidecar_mode + " " + self.compute_uri(self.server.labels["app"], "direct_port")
 
     def bothsidecar(self, load_gen_cmd, sidecar_mode):
-        return load_gen_cmd + "_" + sidecar_mode + " " + self.compute_uri(self.server.labels["app"], "port")
+        return load_gen_cmd + sidecar_mode + " " + self.compute_uri(self.server.labels["app"], "port")
 
     def ingress(self, load_gen_cmd):
         url = urlparse(self.run_ingress)
@@ -200,9 +201,9 @@ def ingress(self, load_gen_cmd):
     def execute_sidecar_mode(self, sidecar_mode, load_gen_type, load_gen_cmd, sidecar_mode_func, labels, perf_label_suffix):
         print('-------------- Running in {sidecar_mode} mode --------------'.format(sidecar_mode=sidecar_mode))
         if load_gen_type == "fortio":
-            kubectl_exec(self.client.name, sidecar_mode_func(load_gen_cmd, sidecar_mode))
+            kubectl_exec(self.client.name, sidecar_mode_func(load_gen_cmd, perf_label_suffix))
         elif load_gen_type == "nighthawk":
-            run_nighthawk(self.client.name, sidecar_mode_func(load_gen_type, sidecar_mode), labels + "_" + sidecar_mode)
+            run_nighthawk(self.client.name, sidecar_mode_func(load_gen_cmd, perf_label_suffix), labels + perf_label_suffix)
 
     def generate_test_labels(self, conn, qps, size):
         size = size or self.size
@@ -304,6 +305,50 @@ def maybe_start_profiling_threads(self, labels, perf_label):
 
         return threads
 
+    def generate_nighthawk_cmd(self, cpus, conn, qps, duration, labels):
+        nighthawk_args = [
+            "nighthawk_client",
+            "--concurrency {cpus}",
+            "--output-format json",
+            "--prefetch-connections",
+            "--open-loop",
+            "--jitter-uniform 0.0001s",
+            "--experimental-h1-connection-reuse-strategy lru",
+            "--experimental-h2-use-multiple-connections",
+            "--nighthawk-service 127.0.0.1:{port_forward}",
+            "--label Nighthawk",
+            "--connections {conn}",
+            "--rps {qps}",
+            "--duration {duration}",
+            "--request-header \"x-nighthawk-test-server-config: {{response_body_size:{size}}}\""
+        ]
+
+        # Our "gRPC" mode actually means:
+        #  - https (see get_protocol_uri_fragment())
+        #  - h2
+        #  - with long running connections
+        #  - Also transfer request body sized according to "size".
+        if self.mode == "grpc":
+            nighthawk_args.append("--h2")
+            if self.size:
+                nighthawk_args.append(
+                    "--request-header \"content-length: {size}\"")
+
+        # Note: Labels is the last arg, and there's stuff depending on that.
+        # watch out when moving it.
+        nighthawk_args.append("--label {labels}")
+
+        # As the worker count acts as a multiplier, we divide by qps/conn by the number of cpu's to spread load accross the workers so the sum of the workers will target the global qps/connection levels.
+        nighthawk_cmd = " ".join(nighthawk_args).format(
+            conn=round(conn / cpus),
+            qps=round(qps / cpus),
+            duration=duration,
+            labels=labels,
+            size=self.size,
+            cpus=cpus,
+            port_forward=NIGHTHAWK_GRPC_SERVICE_PORT_FORWARD)
+
+        return nighthawk_cmd
 
     def run(self, headers, conn, qps, size, duration):
         labels = self.generate_test_labels(conn, qps, size)
@@ -317,7 +362,23 @@ def run(self, headers, conn, qps, size, duration):
             cacert_arg = "-cacert {cacert_path}".format(cacert_path=self.cacert)
 
         headers_cmd = self.generate_headers_cmd(headers)
-        fortio_cmd = self.generate_fortio_cmd(headers_cmd, conn, qps, duration, grpc, cacert_arg, labels)
+
+        if self.load_gen_type == "fortio":
+            load_gen_cmd = self.generate_fortio_cmd(headers_cmd, conn, qps, duration, grpc, cacert_arg, labels)
+        elif self.load_gen_type == "nighthawk":
+            # TODO(oschaaf): Figure out how to best determine the right concurrency for Nighthawk.
+            # Results seem to get very noisy as the number of workers increases, are the clients
+            # and running on separate sets of vCPU cores? nproc yields the same concurrency as goprocs
+            # use with the Fortio version.
+            # client_cpus = int(run_command_sync(
+            #     "kubectl exec -n \"{ns}\" svc/fortioclient -c shell nproc".format(ns=NAMESPACE)))
+            # print("Client pod has {client_cpus} cpus".format(client_cpus=client_cpus))
+
+            # See the comment above, we restrict execution to a single nighthawk worker for
+            # now to avoid noise.
+            workers = 1
+            load_gen_cmd = self.generate_nighthawk_cmd(workers, conn, qps, duration, labels)
+
         perf_label = ""
         sidecar_mode = ""
         sidecar_mode_func = None
@@ -351,9 +412,9 @@ def run(self, headers, conn, qps, size, duration):
 
         if self.run_ingress:
             print('-------------- Running in ingress mode --------------')
-            kubectl_exec(self.client.name, self.ingress(fortio_cmd))
+            kubectl_exec(self.client.name, self.ingress(load_gen_cmd))
         else:
-            self.execute_sidecar_mode(sidecar_mode, self.load_gen_type, fortio_cmd, sidecar_mode_func, labels, perf_label)
+            self.execute_sidecar_mode(sidecar_mode, self.load_gen_type, load_gen_cmd, sidecar_mode_func, labels, perf_label)
 
         if len(threads) > 0:
             if self.custom_profiling_command: