Some C/C++ tests take unreasonable amount of time for what they're doing #12669
Description
Problem:
our C/C++ PR test suites are capped at 1hr runtime (testcases stop to be scheduled after 1hr of runtime) but we are regularly seeing C++ asan and tsan test runs to exceed 120 mins runtime (and more).
Based on our analysis of runtime per-testcase, it seems some C++ tests cases are taking way too long for what they're doing (e.g. 36 min average / 170 min max runtime for a 1-second benchmark).
Here's the data sample, more can be seen in https://data.corp.google.com/sites/qc42qojbor0i/build_duration_pr/
I expect major improvments the amount of tests we will be able to run and reducing the PR tesuite spikes once the top offenders are addressed.
job_name | test_name | avg_duration_mins | max_duration_mins
grpc/ubuntu/pull_request/grpc_cpp_tsan | bins/tsan/json_run_localhost --scenarios_json '{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "threads_per_cq": 3, "server_type": "SYNC_SERVER"}, "num_clients": 0, "client_config": {"security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_SERVER", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3, "load_params": {"closed_loop": {}}, "client_type": "SYNC_CLIENT", "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' GRPC_POLL_STRATEGY=poll-cv | 36.50 | 175.70 |
grpc/ubuntu/pull_request/grpc_cpp_tsan | json_run_localhost:cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure_low_thread_count GRPC_POLL_STRATEGY=epollsig | 33.40 | 100.10 |
grpc/ubuntu/pull_request/grpc_cpp_asan | bins/asan/json_run_localhost --scenarios_json '{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "threads_per_cq": 3, "server_type": "SYNC_SERVER"}, "num_clients": 0, "client_config": {"security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_SERVER", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3, "load_params": {"closed_loop": {}}, "client_type": "SYNC_CLIENT", "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' GRPC_POLL_STRATEGY=epoll1 | 20.50 | 89.40 |
grpc/ubuntu/pull_request/grpc_cpp_tsan | bins/tsan/json_run_localhost --scenarios_json '{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "threads_per_cq": 3, "server_type": "SYNC_SERVER"}, "num_clients": 0, "client_config": {"security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_SERVER", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3, "load_params": {"closed_loop": {}}, "client_type": "SYNC_CLIENT", "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' GRPC_POLL_STRATEGY=epollsig | 29.40 | 52.90 |
grpc/ubuntu/pull_request/grpc_cpp_tsan | bins/tsan/json_run_localhost --scenarios_json '{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "threads_per_cq": 3, "server_type": "SYNC_SERVER"}, "num_clients": 0, "client_config": {"security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_SERVER", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3, "load_params": {"closed_loop": {}}, "client_type": "SYNC_CLIENT", "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' GRPC_POLL_STRATEGY=epoll1 | 11.60 | 43.30
grpc/ubuntu/pull_request/grpc_c_ubsan | bins/ubsan/h2_ssl_cert_test resource_quota_server GRPC_POLL_STRATEGY=poll-cv | 11.30 | 20.80 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test large_metadata GRPC_POLL_STRATEGY=poll-cv | 11.20 | 20.80 |
grpc/ubuntu/pull_request/grpc_c_ubsan | bins/ubsan/h2_ssl_cert_test write_buffering_at_end GRPC_POLL_STRATEGY=poll-cv | 11.40 | 20.80 |
grpc/ubuntu/pull_request/grpc_c_ubsan | bins/ubsan/h2_ssl_cert_test hpack_size GRPC_POLL_STRATEGY=poll-cv | 11.50 | 20.80 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test request_with_flags GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test ping GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test server_finishes_request GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test max_connection_age GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test write_buffering GRPC_POLL_STRATEGY=poll-cv | 11.30 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test workaround_cronet_compression GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test load_reporting_hook GRPC_POLL_STRATEGY=poll-cv | 11.30 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test streaming_error_response GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test cancel_after_client_done GRPC_POLL_STRATEGY=poll-cv | 11.40 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test cancel_in_a_vacuum GRPC_POLL_STRATEGY=poll-cv | 11.30 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test request_with_payload GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test stream_compression_ping_pong_streaming GRPC_POLL_STRATEGY=poll-cv | 11.40 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test max_concurrent_streams GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test connectivity GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_ubsan | bins/ubsan/h2_ssl_cert_test compressed_payload GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test call_creds GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test resource_quota_server GRPC_POLL_STRATEGY=poll-cv | 11.20 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test filter_latency GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90 |
grpc/ubuntu/pull_request/grpc_c_tsan | bins/tsan/h2_ssl_cert_test hpack_size GRPC_POLL_STRATEGY=poll-cv | 11.10 | 18.90