Skip to content

Commit 7ba64dc

Browse files
committed
feat(gax): implement operation-level client request spans (T3)
1 parent e15ae2d commit 7ba64dc

4 files changed

Lines changed: 254 additions & 52 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java‎

Lines changed: 132 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import io.opentelemetry.api.trace.Span;
3636
import io.opentelemetry.api.trace.SpanBuilder;
3737
import io.opentelemetry.api.trace.SpanKind;
38+
import io.opentelemetry.api.trace.StatusCode;
3839
import io.opentelemetry.api.trace.Tracer;
3940
import java.util.HashMap;
4041
import java.util.Map;
@@ -52,22 +53,27 @@ class OpenTelemetryTracingTracer implements ApiTracer {
5253
private final Tracer tracer;
5354
private final Map<String, Object> attemptAttributes;
5455
private final String attemptSpanName;
56+
private final String operationSpanName;
5557
private final ApiTracerContext apiTracerContext;
5658
// Captures the active trace context from the calling thread at RPC initiation.
57-
// This allows attempt spans—including retries dispatched on background threads—to
58-
// link back to the original parent trace.
59+
// This allows the operation span and attempt spans to link back to the caller's trace.
5960
private final io.opentelemetry.context.Context parentContext;
61+
// Trace context containing the operationSpan, serving as the parent for attempt spans.
62+
private final io.opentelemetry.context.Context operationContext;
6063
// Lock coordinates attempt transitions and operation completion across threads.
6164
private final ReentrantLock lock = new ReentrantLock();
6265
private boolean operationCompleted;
63-
// attemptSpan is volatile to ensure fresh reads for thread-safe snapshotting.
66+
// operationSpan and attemptSpan are volatile to ensure fresh reads for thread-safe snapshotting.
67+
private volatile @Nullable Span operationSpan;
6468
private volatile @Nullable Span attemptSpan;
6569

6670
@Override
6771
public void injectTraceContext(java.util.Map<String, String> carrier) {
68-
if (attemptSpan != null) {
72+
Span currentAttempt = attemptSpan;
73+
Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan;
74+
if (spanToInject != null) {
6975
io.opentelemetry.context.Context context =
70-
io.opentelemetry.context.Context.current().with(attemptSpan);
76+
io.opentelemetry.context.Context.current().with(spanToInject);
7177
io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance()
7278
.inject(
7379
context,
@@ -80,19 +86,25 @@ public void injectTraceContext(java.util.Map<String, String> carrier) {
8086
}
8187
}
8288

89+
@Override
90+
public Scope inScope() {
91+
Span currentAttempt = attemptSpan;
92+
Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan;
93+
if (currentSpan == null) {
94+
return () -> {};
95+
}
96+
io.opentelemetry.context.Scope otelScope = currentSpan.makeCurrent();
97+
return otelScope::close;
98+
}
99+
83100
/**
84101
* Creates a new instance of {@code OpenTelemetryTracingTracer}.
85102
*
86103
* @param tracer the {@link Tracer} to use for recording spans
87104
* @param apiTracerContext the {@link ApiTracerContext} to use for recording spans
88105
*/
89106
OpenTelemetryTracingTracer(Tracer tracer, ApiTracerContext apiTracerContext) {
90-
this.tracer = tracer;
91-
this.apiTracerContext = apiTracerContext;
92-
this.attemptSpanName = resolveAttemptSpanName(apiTracerContext);
93-
this.attemptAttributes = new HashMap<>();
94-
this.parentContext = io.opentelemetry.context.Context.current();
95-
buildAttributes();
107+
this(tracer, apiTracerContext, resolveAttemptSpanName(apiTracerContext));
96108
}
97109

98110
/**
@@ -106,12 +118,64 @@ public void injectTraceContext(java.util.Map<String, String> carrier) {
106118
@InternalApi
107119
OpenTelemetryTracingTracer(
108120
Tracer tracer, ApiTracerContext apiTracerContext, String attemptSpanName) {
121+
this(tracer, apiTracerContext, attemptSpanName, resolveOperationSpanName(attemptSpanName));
122+
}
123+
124+
/**
125+
* Creates a new instance of {@code OpenTelemetryTracingTracer} with explicitly provided attempt
126+
* and operation span names.
127+
*
128+
* @param tracer the {@link Tracer} to use for recording spans
129+
* @param apiTracerContext the {@link ApiTracerContext} to use for recording spans
130+
* @param attemptSpanName the name of the individual attempt spans
131+
* @param operationSpanName the name of the overall client request operation span
132+
*/
133+
@InternalApi
134+
OpenTelemetryTracingTracer(
135+
Tracer tracer,
136+
ApiTracerContext apiTracerContext,
137+
String attemptSpanName,
138+
String operationSpanName) {
109139
this.tracer = tracer;
110-
this.attemptSpanName = attemptSpanName;
111140
this.apiTracerContext = apiTracerContext;
141+
this.attemptSpanName = attemptSpanName;
142+
this.operationSpanName = operationSpanName;
112143
this.attemptAttributes = new HashMap<>();
113144
this.parentContext = io.opentelemetry.context.Context.current();
114145
buildAttributes();
146+
this.operationSpan = startOperationSpan();
147+
this.operationContext = parentContext.with(this.operationSpan);
148+
}
149+
150+
/**
151+
* Starts and initializes the operation-level client request span (T3).
152+
*
153+
* @return the newly started {@link Span} for the overall operation
154+
*/
155+
private Span startOperationSpan() {
156+
SpanBuilder operationSpanBuilder = tracer.spanBuilder(operationSpanName);
157+
operationSpanBuilder.setSpanKind(SpanKind.INTERNAL);
158+
operationSpanBuilder.setParent(parentContext);
159+
operationSpanBuilder.setAllAttributes(
160+
ObservabilityUtils.toOtelAttributes(this.attemptAttributes));
161+
return operationSpanBuilder.startSpan();
162+
}
163+
164+
/**
165+
* Derives the operation-level span name from the attempt span name.
166+
*
167+
* @param attemptSpanName the attempt span name
168+
* @return the operation span name
169+
*/
170+
private static String resolveOperationSpanName(String attemptSpanName) {
171+
if (!Strings.isNullOrEmpty(attemptSpanName)) {
172+
if (attemptSpanName.endsWith("/attempt")) {
173+
String name = attemptSpanName.substring(0, attemptSpanName.length() - "/attempt".length());
174+
return name.isEmpty() ? "operation" : name;
175+
}
176+
return "attempt".equals(attemptSpanName) ? "operation" : attemptSpanName;
177+
}
178+
return "operation";
115179
}
116180

117181
private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) {
@@ -139,7 +203,7 @@ public void attemptStarted(Object request, int attemptNumber) {
139203
lock.lock();
140204
try {
141205
// Prevent creating new attempt spans if the overall operation has already concluded.
142-
if (operationCompleted) {
206+
if (operationCompleted || operationSpan == null) {
143207
return;
144208
}
145209
// If a previous attempt was not explicitly closed before a retry started,
@@ -166,8 +230,8 @@ public void attemptStarted(Object request, int attemptNumber) {
166230
// Attempt spans are of the CLIENT kind
167231
spanBuilder.setSpanKind(SpanKind.CLIENT);
168232

169-
// Link attempt span to parent context
170-
spanBuilder.setParent(parentContext);
233+
// Link attempt span to operation context (parent T3 span)
234+
spanBuilder.setParent(operationContext);
171235

172236
// Pass the combined attributes to the new SpanBuilder method
173237
spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes));
@@ -185,35 +249,79 @@ public void attemptStarted(Object request, int attemptNumber) {
185249
/**
186250
* Signals that the overall logical operation succeeded.
187251
*
188-
* <p>Marks the operation as complete and closes any remaining in-flight attempt span.
252+
* <p>Closes any remaining in-flight attempt span and ends the operation span.
189253
*/
190254
@Override
191255
public void operationSucceeded() {
192-
recordErrorAndEndAttempt(null, true);
256+
recordErrorAndEndOperation(null);
193257
}
194258

195259
/**
196260
* Signals that the overall logical operation was cancelled.
197261
*
198-
* <p>Marks the operation as complete and closes any remaining in-flight attempt span with a
199-
* {@link CancellationException}.
262+
* <p>Closes any remaining in-flight attempt span with a {@link CancellationException} and ends
263+
* the operation span with an ERROR status.
200264
*/
201265
@Override
202266
public void operationCancelled() {
203-
recordErrorAndEndAttempt(new CancellationException(), true);
267+
recordErrorAndEndOperation(new CancellationException());
204268
}
205269

206270
/**
207271
* Signals that the overall logical operation failed permanently.
208272
*
209-
* <p>Marks the operation as complete and closes any remaining in-flight attempt span with the
210-
* provided error details.
273+
* <p>Closes any remaining in-flight attempt span and ends the operation span with the provided
274+
* error details and an ERROR status.
211275
*
212276
* @param error the cause of the operation failure
213277
*/
214278
@Override
215279
public void operationFailed(Throwable error) {
216-
recordErrorAndEndAttempt(error, true);
280+
recordErrorAndEndOperation(error);
281+
}
282+
283+
/**
284+
* Records error details and ends both the active attempt span and the operation span in a
285+
* thread-safe manner.
286+
*
287+
* @param error the exception associated with the operation failure, or {@code null} if successful
288+
*/
289+
private void recordErrorAndEndOperation(@Nullable Throwable error) {
290+
Span localOperationSpan;
291+
Span localAttemptSpan;
292+
lock.lock();
293+
try {
294+
operationCompleted = true;
295+
localOperationSpan = operationSpan;
296+
if (localOperationSpan == null) {
297+
return;
298+
}
299+
operationSpan = null;
300+
localAttemptSpan = attemptSpan;
301+
attemptSpan = null;
302+
} finally {
303+
lock.unlock();
304+
}
305+
306+
if (localAttemptSpan != null) {
307+
endAttemptSpan(localAttemptSpan, error);
308+
}
309+
310+
Map<String, Object> responseAttributes =
311+
ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport());
312+
if (!responseAttributes.isEmpty()) {
313+
localOperationSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes));
314+
}
315+
316+
if (error != null) {
317+
localOperationSpan.setStatus(StatusCode.ERROR);
318+
if (!Strings.isNullOrEmpty(error.getMessage())) {
319+
localOperationSpan.setAttribute(
320+
ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage());
321+
}
322+
}
323+
324+
localOperationSpan.end();
217325
}
218326

219327
@Override
@@ -286,24 +394,15 @@ public void attemptPermanentFailure(Throwable error) {
286394
recordErrorAndEndAttempt(error);
287395
}
288396

289-
private void recordErrorAndEndAttempt(@Nullable Throwable error) {
290-
recordErrorAndEndAttempt(error, false);
291-
}
292-
293397
/**
294398
* Records error details and ends the current attempt span in a thread-safe manner.
295399
*
296400
* @param error the exception associated with the attempt failure, or {@code null} if successful
297-
* @param isOperationComplete {@code true} if this call marks the end of the entire logical
298-
* operation, preventing subsequent retry attempts from starting
299401
*/
300-
private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) {
402+
private void recordErrorAndEndAttempt(@Nullable Throwable error) {
301403
Span localAttemptSpan;
302404
lock.lock();
303405
try {
304-
if (isOperationComplete) {
305-
operationCompleted = true;
306-
}
307406
localAttemptSpan = attemptSpan;
308407
if (localAttemptSpan == null) {
309408
return;

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ void testNewTracer_withContext_grpc_usesFullMethodName() {
229229

230230
tracerInstance.attemptStarted(null, 1);
231231

232-
verify(tracer).spanBuilder("google.cloud.v1.Service/Method");
232+
verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service/Method");
233233
}
234234

235235
@ParameterizedTest
@@ -256,7 +256,7 @@ void testNewTracer_withContext_http_usesHttpMethodAndPathTemplate(
256256

257257
tracerInstance.attemptStarted(null, 1);
258258

259-
verify(tracer).spanBuilder(expectedSpanName);
259+
verify(tracer, atLeastOnce()).spanBuilder(expectedSpanName);
260260
}
261261

262262
@Test
@@ -274,7 +274,7 @@ void testNewTracer_withContext_http_noHttpMethodOrPathTemplate_usesFullMethodNam
274274

275275
tracerInstance.attemptStarted(null, 1);
276276

277-
verify(tracer).spanBuilder("google.cloud.v1.Service.Method");
277+
verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service.Method");
278278
}
279279

280280
@Test
@@ -310,7 +310,7 @@ void testNewTracer_mergesFactoryContext() {
310310
tracerInstance.attemptStarted(null, 1);
311311

312312
ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
313-
verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
313+
verify(spanBuilder, atLeastOnce()).setAllAttributes(attributesCaptor.capture());
314314

315315
Attributes attributes = attributesCaptor.getValue();
316316
assertThat(attributes.asMap())

0 commit comments

Comments
 (0)