shivanee-p · Oct 3, 2026 · Oct 1, 2026 · Oct 1, 2026 · Oct 1, 2026 · Oct 1, 2026
@@ -17,17 +17,12 @@
 // [START gax_observability]
 'use strict';
 
-// 1. INITIALIZE OPENTELEMETRY BEFORE IMPORTING ANY CLIENT LIBRARIES
-// In Node.js, instrumentations must patch the networking modules (http, grpc)
-// before any Google Cloud client libraries are loaded into the module cache.
+// 1. IMPORT OPENTELEMETRY MODULES
 const {NodeTracerProvider} = require('@opentelemetry/sdk-trace-node');
 const {BatchSpanProcessor} = require('@opentelemetry/sdk-trace-base');
 const {
   TraceExporter,
 } = require('@google-cloud/opentelemetry-cloud-trace-exporter');
-const {registerInstrumentations} = require('@opentelemetry/instrumentation');
-const {HttpInstrumentation} = require('@opentelemetry/instrumentation-http');
-const {GrpcInstrumentation} = require('@opentelemetry/instrumentation-grpc');
 
 // 2. CONFIGURE TRACING: SET UP A TRACER PROVIDER AND EXPORTER
 const cloudTraceExporter = new TraceExporter();
@@ -38,27 +33,19 @@ const provider = new NodeTracerProvider({
 });
 provider.register();
 
-// 3. ENABLE LOW-LEVEL NETWORK TRACING SPANS USING INSTRUMENTATION LIBRARIES
-registerInstrumentations({
-  instrumentations: [
-    new HttpInstrumentation(),
-    new GrpcInstrumentation(),
-  ],
-});
-
-// 4. ENABLE CLIENT REQUEST TRACING SPANS WITH ENV VARIABLE
+// 3. ENABLE TRACING SPANS WITH ENV VARIABLE
 // Sets the flag before client libraries or RPC callers initialize
 process.env.GOOGLE_SDK_NODE_ENABLE_TRACING = 'true';
 
-// 5. IMPORT CLIENT LIBRARIES AFTER OPENTELEMETRY SETUP
+// 4. IMPORT CLIENT LIBRARIES AFTER OPENTELEMETRY SETUP
 // Replace with your Google Cloud client library, for example:
 // const { SecretManagerServiceClient } = require('@google-cloud/secret-manager');
 
 async function main() {
   // const client = new SecretManagerServiceClient();
   // await client.listSecrets({parent: 'projects/my-project'});
 
-  // 6. FLUSH SPANS BEFORE PROCESS EXIT
+  // 5. FLUSH SPANS BEFORE PROCESS EXIT
   // Ensures all buffered spans in BatchSpanProcessor are exported to Cloud Trace
   await provider.forceFlush();
   console.log('Tracing initialized successfully.');

@@ -18,12 +18,14 @@
  * Provides function wrappers that implement page streaming and retrying.
  */
 
+import {context} from '@opentelemetry/api';
 import {createAPICaller} from './apiCaller';
 import {
   APICallback,
   GaxCall,
   GRPCCall,
   GRPCCallOtherArgs,
+  GRPCCallResult,
   RequestType,
   SimpleCallbackFunction,
 } from './apitypes';
@@ -36,8 +38,10 @@ import {StreamProxy} from './streamingCalls/streaming';
 import {warn} from './warnings';
 import {
   traceCall,
+  traceAttempt,
   StaticTraceContext,
   DynamicTraceContext,
+  AttemptTraceContext,
   ResendRecorder,
 } from './observability/TracerHelper';
 import {resolveStaticTraceContext} from './observability/metadataResolver';
@@ -77,13 +81,35 @@ export function createApiCall(
   const apiCaller = createAPICaller(settings, descriptor);
 
   const tracingEnabled = checkTelemetryEnabled(settings);
+  const staticArgs: StaticTraceContext | undefined = tracingEnabled
+    ? resolveStaticTraceContext(settings)
+    : undefined;
+  const serviceName = settings.apiName?.split('.').pop() ?? '';
+  const isFallback = Boolean(_fallback);
+  const dynamicArgs: DynamicTraceContext | undefined = tracingEnabled
+    ? {
+        clientName: serviceName ? `${serviceName}Client` : '',
+        methodName: settings.otherArgs?.internalMethodName ?? '',
+        rpcType: isFallback ? 'http' : 'grpc',
+      }
+    : undefined;
+  // Context for per-attempt low level network CLIENT spans.
+  const attemptDynamicArgs: AttemptTraceContext | undefined =
+    tracingEnabled && dynamicArgs
+      ? {
+          ...dynamicArgs,
+          apiName: settings.apiName ?? '',
+        }
+      : undefined;
 
   const invokeCall = (
     request: RequestType,
     callOptions?: CallOptions,
     callback?: APICallback,
     recordResend?: ResendRecorder,
   ) => {
+    // Capture the active client request span context to parent async low level network attempt spans.
+    const parentContext = tracingEnabled ? context.active() : undefined;
     let currentApiCaller = apiCaller;
 
     let thisSettings: CallSettings;
@@ -108,6 +134,19 @@ export function createApiCall(
 
     const ongoingCall = currentApiCaller.init(callback);
 
+    // Track resends per logical request so auto-paginated subsequent page
+    // requests start at resendCount = 0 instead of being counted as retries.
+    let attemptResendCount = 0;
+    let isRetryAttempt = false;
+    const originalRecordResend = recordResend;
+    if (tracingEnabled) {
+      recordResend = () => {
+        originalRecordResend?.();
+        attemptResendCount++;
+        isRetryAttempt = true;
+      };
+    }
+
     // Server-streaming calls retry inside the stream rather than through
     // `retryable`, so the recorder is handed to the stream itself. It is the
     // same span either way: the proxy outlives its resumptions, so the call
@@ -119,11 +158,42 @@ export function createApiCall(
     funcPromise
       .then((func: GRPCCall) => {
         // Initially, the function is just what gRPC server stub contains.
-        func = currentApiCaller.wrap(func);
+        let wrappedFunc = currentApiCaller.wrap(func);
 
         const streaming = (currentApiCaller as StreamingApiCaller).descriptor
           ?.streaming;
 
+        // Wrap the transport call so each attempt (initial send and retries) emits a low level network CLIENT span.
+        if (tracingEnabled && attemptDynamicArgs && staticArgs) {
+          const callerWrappedFunc = wrappedFunc;
+          wrappedFunc = (
+            argument: {},
+            metadata: {},
+            options: {},
+            attemptCallback: APICallback,
+          ): GRPCCallResult => {
+            if (!isRetryAttempt) {
+              attemptResendCount = 0;
+            }
+            isRetryAttempt = false;
+            const resendCount = attemptResendCount;
+            return traceAttempt(
+              {...attemptDynamicArgs, resendCount},
+              staticArgs,
+              tracedAttemptCallback =>
+                Reflect.apply(callerWrappedFunc, undefined, [
+                  argument,
+                  metadata,
+                  options,
+                  tracedAttemptCallback ?? attemptCallback,
+                ]),
+              Boolean(streaming),
+              attemptCallback,
+              parentContext,
+            );
+          };
+        }
+
         const retry = thisSettings.retry;
 
         if (streaming && retry) {
@@ -159,7 +229,7 @@ export function createApiCall(
             retry.backoffSettings.initialRpcTimeoutMillis ??=
               thisSettings.timeout;
             return retryable(
-              func,
+              wrappedFunc,
               thisSettings.retry!,
               thisSettings.otherArgs as GRPCCallOtherArgs,
               thisSettings.apiName,
@@ -168,7 +238,7 @@ export function createApiCall(
           }
         }
         return addTimeoutArg(
-          func,
+          wrappedFunc,
           thisSettings.timeout,
           thisSettings.otherArgs as GRPCCallOtherArgs,
         );
@@ -192,16 +262,7 @@ export function createApiCall(
     return currentApiCaller.result(ongoingCall);
   };
 
-  if (tracingEnabled) {
-    const staticArgs: StaticTraceContext = resolveStaticTraceContext(settings);
-
-    const serviceName = settings.apiName?.split('.').pop() ?? '';
-    const isFallback = Boolean(_fallback);
-    const dynamicArgs: DynamicTraceContext = {
-      clientName: serviceName ? `${serviceName}Client` : '',
-      methodName: settings.otherArgs?.internalMethodName ?? '',
-      rpcType: isFallback ? 'http' : 'grpc',
-    };
+  if (tracingEnabled && dynamicArgs && staticArgs) {
     const isStreamingCall = apiCaller instanceof StreamingApiCaller;
     return (
       request: RequestType,

@@ -108,6 +108,8 @@ export class GrpcClient {
   httpRules?: Array<google.api.IHttpRule>;
   numericEnums: boolean;
   minifyJson: boolean;
+  private _servicePath?: string;
+  private _port?: number;
 
   /**
    * In rare cases users might need to deallocate all memory consumed by loaded protos.
@@ -156,6 +158,10 @@ export class GrpcClient {
     this.httpRules = (options as GrpcClientOptions).httpRules;
     this.numericEnums = (options as GrpcClientOptions).numericEnums ?? false;
     this.minifyJson = (options as GrpcClientOptions).minifyJson ?? false;
+    this._servicePath =
+      (options as GrpcClientOptions).servicePath ||
+      (options as GrpcClientOptions).apiEndpoint;
+    this._port = (options as GrpcClientOptions).port;
   }
 
   /**
@@ -279,12 +285,21 @@ export class GrpcClient {
       }
       return metadata;
     }
+    const otherArgs: Record<string, unknown> = {
+      metadataBuilder: buildMetadata,
+    };
+    if (this._servicePath) {
+      otherArgs.servicePath = this._servicePath;
+    }
+    if (this._port !== undefined) {
+      otherArgs.port = this._port;
+    }
     return gax.constructSettings(
       serviceName,
       clientConfig,
       configOverrides,
       Status,
-      {metadataBuilder: buildMetadata},
+      otherArgs,
       enableTelemetryTracing,
       internalTelemetryInfo,
     );

@@ -21,7 +21,7 @@ import {defaultToObjectOptions} from './fallback';
 import {FetchParameters, FetchParametersMethod} from './fallbackServiceStub';
 import {GoogleError} from './googleError';
 import {rpcCodeFromHttpStatusCode} from './status';
-import {transcode} from './transcoding';
+import {getMatchedUrlTemplate, transcode} from './transcoding';
 
 export function encodeRequest(
   rpc: protobuf.Method,
@@ -82,6 +82,7 @@ export function encodeRequest(
   return {
     method,
     url,
+    urlTemplate: getMatchedUrlTemplate(transcoded),
     headers,
     body,
   };

@@ -25,6 +25,7 @@
 import {StreamArrayParser} from './streamArrayParser';
 import {defaultToObjectOptions} from './fallback';
 import {GoogleError} from './googleError';
+import {setAttemptHttpMethod} from './observability/TracerHelper';
 import {rpcCodeFromHttpStatusCode, Status} from './status';
 import {pipeline, PipelineSource} from 'stream';
 import type {Agent as HttpAgent} from 'http';
@@ -68,6 +69,7 @@
   body: Buffer | Uint8Array | string;
   method: FetchParametersMethod;
   url: string;
+  urlTemplate?: string;
 }
 
 // helper function used to properly format empty responses
@@ -302,6 +304,8 @@
         };
       }
 
+      setAttemptHttpMethod(fetchParameters.method, fetchParameters.urlTemplate);
+
       const cancelController = new AbortController();
       const cancelSignal = cancelController.signal as AbortSignal;
       let cancelRequested = false;
@@ -433,7 +437,7 @@
                // state, as the handlers below do.
                if (err && (timedOut || !cancelRequested)) {
                  if (callback) {
                    callback(err);
                  }
                  streamArrayParser.emit('error', err);
                }
@@ -448,7 +452,7 @@
              Promise.resolve(response.ok),
              response.arrayBuffer(),
            ])
              .then(([ok, buffer]: [boolean, Buffer | ArrayBuffer]) => {
                const response = responseDecoder(
                  rpc,
                  ok,
@@ -458,7 +462,7 @@
                callback!(null, response);
                return;
              })
              .catch((err: Error) => {
                // The deadline can expire after the response headers arrive but
                // before the body is fully read, which rejects here rather than
                // in the outer handler.
@@ -481,7 +485,7 @@
                  // state we recorded.
                  if (timedOut || !cancelRequested) {
                    if (callback) {
                      callback(callErr);
                    }
                    streamArrayParser.emit('error', callErr);
                  }
@@ -539,12 +543,12 @@
            // nobody is listening to any more.
            if (timedOut || !cancelRequested) {
              if (callback) {
                callback(err);
              }
              streamArrayParser.emit('error', err);
            }
          } else if (callback) {
            callback(err);
          } else {
            throw err;
          }

@@ -53,6 +53,9 @@ export interface GrpcClientOptions extends GoogleAuthOptions {
   httpRules?: Array<google.api.IHttpRule>;
   numericEnums?: boolean;
   universeDomain?: string;
+  servicePath?: string;
+  apiEndpoint?: string;
+  port?: number;
 }
 
 export interface MetadataValue {
@@ -121,6 +124,8 @@ export class GrpcClient {
   fallback: boolean | 'rest' | 'proto';
   private static protoCache = new Map<string, grpc.GrpcObject>();
   httpRules?: Array<google.api.IHttpRule>;
+  private _servicePath?: string;
+  private _port?: number;
   /**
    * Base directory for resolving client certificates.
    *
@@ -174,6 +179,8 @@ export class GrpcClient {
   constructor(options: GrpcClientOptions = {}) {
     this.auth = options.auth || new GoogleAuth(options);
     this.fallback = false;
+    this._servicePath = options.servicePath || options.apiEndpoint;
+    this._port = options.port;
 
     const minimumVersion = 10;
     const major = Number(process.version.match(/^v(\d+)/)?.[1]);
@@ -388,12 +395,21 @@ export class GrpcClient {
     enableTelemetryTracing?: boolean,
     internalTelemetryInfo?: StaticTraceContext,
   ) {
+    const otherArgs: Record<string, unknown> = {
+      metadataBuilder: this.metadataBuilder(headers),
+    };
+    if (this._servicePath) {
+      otherArgs.servicePath = this._servicePath;
+    }
+    if (this._port !== undefined) {
+      otherArgs.port = this._port;
+    }
     return gax.constructSettings(
       serviceName,
       clientConfig,
       configOverrides,
       this.grpc.status,
-      {metadataBuilder: this.metadataBuilder(headers)},
+      otherArgs,
       enableTelemetryTracing,
       internalTelemetryInfo,
     );