From 1e5a18c9d5d987b60b804beabbd11d675a523421 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 2 Jul 2026 18:50:23 +0000 Subject: [PATCH 1/4] fix(datastore): reduce flakiness of E2E tracing tests Fixes https://github.com/googleapis/google-cloud-java/issues/13644 Increase the trace retry count to 120 (from 60) to allow more time for eventual consistency in Cloud Trace. Log a warning if forceFlush() does not complete successfully. Shutdown SdkTracerProvider in after() to ensure clean resource release. TAG=agy CONV=c8697e3d-c06b-4d5b-86e9-f8950e4298fd --- .../google/cloud/datastore/it/ITE2ETracingTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 63a80fe1913f..6e7c6ad67934 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -227,7 +227,7 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static final int NUM_SPAN_ID_BYTES = 16; - private static final int GET_TRACE_RETRY_COUNT = 60; + private static final int GET_TRACE_RETRY_COUNT = 120; private static final int GET_TRACE_RETRY_BACKOFF_MILLIS = 1000; @@ -393,6 +393,12 @@ public void after() throws Exception { tracer = null; retrievedTrace = null; customSpanContext = null; + if (openTelemetrySdk != null) { + openTelemetrySdk + .getSdkTracerProvider() + .shutdown() + .join(TRACE_PROVIDER_SHUTDOWN_MILLIS, TimeUnit.MILLISECONDS); + } openTelemetrySdk = null; } @@ -441,6 +447,9 @@ protected void waitForTracesToComplete() throws Exception { CompletableResultCode completableResultCode = openTelemetrySdk.getSdkTracerProvider().forceFlush(); completableResultCode.join(TRACE_FORCE_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + if (!completableResultCode.isSuccess()) { + logger.warning("Force flush did not complete successfully"); + } } // Validates `retrievedTrace`. Cloud Trace indexes traces w/ eventual consistency, even when From 9493964b40cee58ae22551a0a4229787a8bda3ca Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 6 Jul 2026 14:55:38 +0000 Subject: [PATCH 2/4] fix(datastore): use openTelemetrySdk.close() in teardown Use openTelemetrySdk.close() instead of manually shutting down the tracer provider, as suggested by the reviewer. This is more robust and idiomatic. Also remove the now unused constant TRACE_PROVIDER_SHUTDOWN_MILLIS. TAG=agy CONV=c8697e3d-c06b-4d5b-86e9-f8950e4298fd --- .../com/google/cloud/datastore/it/ITE2ETracingTest.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 6e7c6ad67934..4219863e5c93 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -233,8 +233,6 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static final int TRACE_FORCE_FLUSH_MILLIS = 5000; - private static final int TRACE_PROVIDER_SHUTDOWN_MILLIS = 1000; - private static Key KEY1; private static Key KEY2; @@ -394,10 +392,7 @@ public void after() throws Exception { retrievedTrace = null; customSpanContext = null; if (openTelemetrySdk != null) { - openTelemetrySdk - .getSdkTracerProvider() - .shutdown() - .join(TRACE_PROVIDER_SHUTDOWN_MILLIS, TimeUnit.MILLISECONDS); + openTelemetrySdk.close(); } openTelemetrySdk = null; } From 91b9e21164ee2e58291717e9846417d1f7ce45b6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 6 Jul 2026 18:32:48 +0000 Subject: [PATCH 3/4] fix(datastore): make tracer and SDK fields non-static in E2E tests Make openTelemetrySdk, traceExporter, tracer, and other tracing-related fields non-static to avoid sharing them (and closing them) across tests, which was causing "Channel shutdown invoked" errors. Store credentials statically since they can be shared safely. Initialize traceExporter in before() instead of setup(). TAG=agy CONV=c8697e3d-c06b-4d5b-86e9-f8950e4298fd --- .../cloud/datastore/it/ITE2ETracingTest.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 4219863e5c93..362e2b0c29dc 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -244,22 +244,24 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack // Random int generator for trace ID and span ID private static Random random; - private static TraceExporter traceExporter; + private static Credentials credentials; + + private TraceExporter traceExporter; // Required for reading back traces from Cloud Trace for validation private static TraceServiceClient traceClient; // Custom SpanContext for each test, required for TraceID injection - private static SpanContext customSpanContext; + private SpanContext customSpanContext; // Trace read back from Cloud Trace using traceClient for verification - private static Trace retrievedTrace; + private Trace retrievedTrace; - private static String rootSpanName; - private static Tracer tracer; + private String rootSpanName; + private Tracer tracer; // Required to set custom-root span - private static OpenTelemetrySdk openTelemetrySdk; + private OpenTelemetrySdk openTelemetrySdk; private static String projectId; @@ -285,14 +287,7 @@ public static void setup() throws IOException { // Share the same credentials used by Datastore client with the TraceExporter and // TraceServiceClient to ensure consistency and avoid auth issues in environments // where default ADC resolution might fail for the exporter. - Credentials credentials = DatastoreOptions.getDefaultInstance().getCredentials(); - - TraceConfiguration.Builder traceConfigurationBuilder = - TraceConfiguration.builder().setProjectId(projectId); - if (credentials != null) { - traceConfigurationBuilder.setCredentials(credentials); - } - traceExporter = TraceExporter.createWithConfiguration(traceConfigurationBuilder.build()); + credentials = DatastoreOptions.getDefaultInstance().getCredentials(); TraceServiceSettings.Builder clientBuilder = TraceServiceSettings.newBuilder(); if (credentials != null) { @@ -308,6 +303,13 @@ public void before() throws Exception { Resource resource = Resource.getDefault().merge(Resource.builder().put(SERVICE_NAME, "Sparky").build()); + TraceConfiguration.Builder traceConfigurationBuilder = + TraceConfiguration.builder().setProjectId(projectId); + if (credentials != null) { + traceConfigurationBuilder.setCredentials(credentials); + } + traceExporter = TraceExporter.createWithConfiguration(traceConfigurationBuilder.build()); + if (isUsingGlobalOpenTelemetrySDK()) { openTelemetrySdk = OpenTelemetrySdk.builder() From 6d017815f5b145edf7034005e167d94fb29ebe8e Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 6 Jul 2026 18:46:42 +0000 Subject: [PATCH 4/4] fix(datastore): address PR comments on E2E tracing tests - Shut down traceExporter in after() to avoid resource leaks. - Add null check for openTelemetrySdk in waitForTracesToComplete() to avoid NPE. TAG=agy CONV=c8697e3d-c06b-4d5b-86e9-f8950e4298fd --- .../cloud/datastore/it/ITE2ETracingTest.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 362e2b0c29dc..dfaa2aff6d76 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -393,10 +393,22 @@ public void after() throws Exception { tracer = null; retrievedTrace = null; customSpanContext = null; - if (openTelemetrySdk != null) { - openTelemetrySdk.close(); + try { + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } + } finally { + if (traceExporter != null) { + try { + // Attempt to shut down traceExporter. + traceExporter.shutdown(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to shut down traceExporter", e); + } + } } openTelemetrySdk = null; + traceExporter = null; } @AfterClass @@ -440,6 +452,10 @@ protected Span getNewRootSpanWithContext() { } protected void waitForTracesToComplete() throws Exception { + if (openTelemetrySdk == null) { + logger.warning("OpenTelemetrySdk is null, cannot flush traces"); + return; + } logger.info("Flushing traces..."); CompletableResultCode completableResultCode = openTelemetrySdk.getSdkTracerProvider().forceFlush();