Add grpc request and response metadata (#3386)

Fixes #3358

experimental/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to experimental packages in this project will be documented
 ### :rocket: (Enhancement)
 
 * feat(instrumentation-http): monitor error events with events.errorMonitor [#3402](https://github.com/open-telemetry/opentelemetry-js/pull/3402) @legendecas
+* feat(instrumentation-grpc): added grpc metadata client side attributes in instrumentation [#3386](https://github.com/open-telemetry/opentelemetry-js/pull/3386)
 * feat(instrumentation): add new `_setMeterInstruments` protected method that update the meter instruments every meter provider update.
 
 ### :bug: (Bug Fix)

experimental/packages/opentelemetry-instrumentation-grpc/README.md

@@ -44,9 +44,10 @@ See [examples/grpc](https://github.com/open-telemetry/opentelemetry-js/tree/main
 
 gRPC instrumentation accepts the following configuration:
 
-| Options | Type | Description |
-| ------- | ---- | ----------- |
-| [`ignoreGrpcMethods`](https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-instrumentation-grpc/src/types.ts#L25) | `IgnoreMatcher[]` | gRPC instrumentation will not trace any methods that match anything in this list. You may pass a string (case-insensitive match), a `RegExp` object, or a filter function. |
+| Options                                                                                                                                                              | Type              | Description                                                                                                                                                                                                                                                               |
+|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [`ignoreGrpcMethods`](https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-instrumentation-grpc/src/types.ts#L25)        | `IgnoreMatcher[]` | gRPC instrumentation will not trace any methods that match anything in this list. You may pass a string (case-insensitive match), a `RegExp` object, or a filter function.                                                                                                |
+| [`metadataToSpanAttributes`](https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-instrumentation-grpc/src/types.ts#L27) | `object`          | List of case insensitive metadata to convert to span attributes. Client (outgoing requests, incoming responses) metadata attributes will be converted to span attributes in the form of `rpc.{request\response}.metadata.metadata_key`, e.g. `rpc.response.metadata.date` |
 
 ## Useful links
 

experimental/packages/opentelemetry-instrumentation-grpc/src/grpc-js/clientUtils.ts

@@ -33,6 +33,7 @@ import { CALL_SPAN_ENDED } from './serverUtils';
 import { EventEmitter } from 'events';
 import { AttributeNames } from '../enums/AttributeNames';
 import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
+import { metadataCaptureType } from '../types';
 
 /**
  * Parse a package method list and return a list of methods to patch
@@ -69,6 +70,7 @@ export function getMethodsToWrap(
  * span on callback or receiving an emitted event.
  */
 export function makeGrpcClientRemoteCall(
+  metadataCapture: metadataCaptureType,
   original: GrpcClientFunc,
   args: unknown[],
   metadata: grpcJs.Metadata,
@@ -129,6 +131,10 @@ export function makeGrpcClientRemoteCall(
     setSpanContext(metadata);
     const call = original.apply(self, args);
 
+    call.on('metadata', responseMetadata => {
+      metadataCapture.client.captureResponseMetadata(span, responseMetadata);
+    });
+
     // if server stream or bidi
     if (original.responseStream) {
       // Both error and status events can be emitted

experimental/packages/opentelemetry-instrumentation-grpc/src/grpc-js/index.ts

@@ -20,7 +20,7 @@ import {
   isWrapped,
 } from '@opentelemetry/instrumentation';
 import { InstrumentationBase } from '@opentelemetry/instrumentation';
-import { GrpcInstrumentationConfig } from '../types';
+import { GrpcInstrumentationConfig, metadataCaptureType } from '../types';
 import {
   ServerCallWithMeta,
   SendUnaryDataCallback,
@@ -49,17 +49,20 @@ import {
   getMetadata,
 } from './clientUtils';
 import { EventEmitter } from 'events';
-import { _extractMethodAndService } from '../utils';
+import {_extractMethodAndService, metadataCapture} from '../utils';
 import { AttributeValues } from '../enums/AttributeValues';
 import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
 
 export class GrpcJsInstrumentation extends InstrumentationBase {
+  private _metadataCapture: metadataCaptureType;
+
   constructor(
     name: string,
     version: string,
     config?: GrpcInstrumentationConfig,
   ) {
     super(name, version, config);
+    this._metadataCapture = this._createMetadataCapture();
   }
 
   init() {
@@ -122,6 +125,11 @@ export class GrpcJsInstrumentation extends InstrumentationBase {
     return super.getConfig();
   }
 
+  override setConfig(config?: GrpcInstrumentationConfig): void {
+    super.setConfig(config);
+    this._metadataCapture = this._createMetadataCapture();
+  }
+
   /**
    * Patch for grpc.Server.prototype.register(...) function. Provides auto-instrumentation for
    * client_stream, server_stream, bidi, unary server handler calls.
@@ -298,8 +306,11 @@ export class GrpcJsInstrumentation extends InstrumentationBase {
             [SemanticAttributes.RPC_METHOD]: method,
             [SemanticAttributes.RPC_SERVICE]: service,
           });
+
+        instrumentation._metadataCapture.client.captureRequestMetadata(span, metadata);
+
         return context.with(trace.setSpan(context.active(), span), () =>
-          makeGrpcClientRemoteCall(original, args, metadata, this)(span)
+          makeGrpcClientRemoteCall(instrumentation._metadataCapture, original, args, metadata, this)(span)
         );
       }
       Object.assign(clientMethodTrace, original);
@@ -334,4 +345,15 @@ export class GrpcJsInstrumentation extends InstrumentationBase {
       }
     });
   }
+
+  private _createMetadataCapture(): metadataCaptureType {
+    const config = this.getConfig();
+
+    return {
+      client: {
+        captureRequestMetadata: metadataCapture('request', config.metadataToSpanAttributes?.client?.requestMetadata ?? []),
+        captureResponseMetadata: metadataCapture('response', config.metadataToSpanAttributes?.client?.responseMetadata ?? [])
+      }
+    };
+  }
 }

experimental/packages/opentelemetry-instrumentation-grpc/src/grpc/clientUtils.ts

@@ -31,11 +31,13 @@ import {
   findIndex,
 } from '../utils';
 import { AttributeNames } from '../enums/AttributeNames';
+import { metadataCaptureType } from '../types';
 
 /**
  * This method handles the client remote call
  */
 export const makeGrpcClientRemoteCall = function (
+  metadataCapture: metadataCaptureType,
   grpcClient: typeof grpcTypes,
   original: GrpcClientFunc,
   args: any[],
@@ -102,6 +104,12 @@ export const makeGrpcClientRemoteCall = function (
     setSpanContext(metadata);
     const call = original.apply(self, args);
 
+    ((call as unknown) as events.EventEmitter).on(
+      'metadata',
+      responseMetadata => {
+        metadataCapture.client.captureResponseMetadata(span, responseMetadata);
+      });
+
     // if server stream or bidi
     if (original.responseStream) {
       // Both error and status events can be emitted

experimental/packages/opentelemetry-instrumentation-grpc/src/grpc/index.ts

@@ -27,7 +27,7 @@ import {
   SendUnaryDataCallback,
   GrpcClientFunc,
 } from './types';
-import { GrpcInstrumentationConfig } from '../types';
+import { GrpcInstrumentationConfig, metadataCaptureType } from '../types';
 import {
   context,
   propagation,
@@ -41,7 +41,7 @@ import {
   serverStreamAndBidiHandler,
 } from './serverUtils';
 import { makeGrpcClientRemoteCall, getMetadata } from './clientUtils';
-import { _extractMethodAndService, _methodIsIgnored } from '../utils';
+import {_extractMethodAndService, _methodIsIgnored, metadataCapture} from '../utils';
 import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
 import {AttributeValues} from '../enums/AttributeValues';
 
@@ -54,12 +54,15 @@ let grpcClient: typeof grpcTypes;
 export class GrpcNativeInstrumentation extends InstrumentationBase<
   typeof grpcTypes
   > {
+  private _metadataCapture: metadataCaptureType;
+
   constructor(
     name: string,
     version: string,
     config?: GrpcInstrumentationConfig
   ) {
     super(name, version, config);
+    this._metadataCapture = this._createMetadataCapture();
   }
 
   init() {
@@ -105,6 +108,11 @@ export class GrpcNativeInstrumentation extends InstrumentationBase<
     return super.getConfig();
   }
 
+  override setConfig(config?: GrpcInstrumentationConfig): void {
+    super.setConfig(config);
+    this._metadataCapture = this._createMetadataCapture();
+  }
+
   private _getInternalPatchs() {
     const onPatch = (
       moduleExports: GrpcInternalClientTypes,
@@ -306,8 +314,12 @@ export class GrpcNativeInstrumentation extends InstrumentationBase<
             [SemanticAttributes.RPC_METHOD]: method,
             [SemanticAttributes.RPC_SERVICE]: service,
           });
+
+        instrumentation._metadataCapture.client.captureRequestMetadata(span, metadata);
+
         return context.with(trace.setSpan(context.active(), span), () =>
           makeGrpcClientRemoteCall(
+            instrumentation._metadataCapture,
             grpcClient,
             original,
             args,
@@ -320,4 +332,15 @@ export class GrpcNativeInstrumentation extends InstrumentationBase<
       return clientMethodTrace;
     };
   }
+
+  private _createMetadataCapture(): metadataCaptureType {
+    const config = this.getConfig();
+
+    return {
+      client: {
+        captureRequestMetadata: metadataCapture('request', config.metadataToSpanAttributes?.client?.requestMetadata ?? []),
+        captureResponseMetadata: metadataCapture('response', config.metadataToSpanAttributes?.client?.responseMetadata ?? [])
+      }
+    };
+  }
 }

experimental/packages/opentelemetry-instrumentation-grpc/src/types.ts

@@ -15,6 +15,9 @@
  */
 
 import { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { Span } from '@opentelemetry/api';
+import type * as grpcJsTypes from '@grpc/grpc-js';
+import type * as grpcTypes from 'grpc';
 
 export type IgnoreMatcher = string | RegExp | ((str: string) => boolean);
 
@@ -23,4 +26,18 @@ export interface GrpcInstrumentationConfig extends InstrumentationConfig {
    * the IgnoreMatchers in the ignoreGrpcMethods list
    */
   ignoreGrpcMethods?: IgnoreMatcher[];
+  /** Map the following gRPC metadata to span attributes. */
+  metadataToSpanAttributes?: {
+    client?: {
+      responseMetadata?: string[],
+      requestMetadata?: string[];
+    }
+  }
 }
+
+export type metadataCaptureType = {
+  client: {
+    captureRequestMetadata: (span: Span, metadata: grpcJsTypes.Metadata | grpcTypes.Metadata) => void,
+    captureResponseMetadata: (span: Span, metadata: grpcJsTypes.Metadata | grpcTypes.Metadata) => void
+  }
+};

experimental/packages/opentelemetry-instrumentation-grpc/src/utils.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { SpanStatusCode, SpanStatus } from '@opentelemetry/api';
+import {SpanStatusCode, SpanStatus, Span} from '@opentelemetry/api';
 import type * as grpcTypes from 'grpc';
 import type * as grpcJsTypes from '@grpc/grpc-js';
 import { IgnoreMatcher } from './types';
@@ -110,3 +110,28 @@ export const _extractMethodAndService = (name: string): { service: string, metho
     method
   });
 };
+
+
+export function metadataCapture(type: 'request' | 'response', metadataToAdd: string[]) {
+  const normalizedMetadataAttributes = new Map(metadataToAdd.map(value => [value.toLowerCase(), value.toLowerCase().replace(/-/g, '_')]));
+
+  return (span: Span, metadata: grpcJsTypes.Metadata | grpcTypes.Metadata) => {
+    for (const [capturedMetadata, normalizedMetadata] of normalizedMetadataAttributes) {
+      const metadataValues =
+        metadata
+          .get(capturedMetadata)
+          .flatMap(value =>
+            typeof value === 'string' ? value.toString() : []
+          );
+
+      if (metadataValues === undefined || metadataValues === []) {
+        continue;
+      }
+
+      const key = `rpc.${type}.metadata.${normalizedMetadata}`;
+
+      span.setAttribute(key, metadataValues);
+    }
+  };
+}
+

experimental/packages/opentelemetry-instrumentation-grpc/test/fixtures/grpc-test.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package pkg_test;
 
 service GrpcTester {
+  rpc unaryMethodWithMetadata (TestRequest) returns (TestReply) {}
   rpc UnaryMethod (TestRequest) returns (TestReply) {}
   rpc camelCaseMethod (TestRequest) returns (TestReply) {}
   rpc ClientStreamMethod (stream TestRequest) returns (TestReply) {}