[Monitor OpenTelemetry Exporter] Fix Property Truncation (#31004)

### Packages impacted by this PR @azure/monitor-opentelemetry-exporter ### Issues associated with this PR #30969 ### Describe the problem that is addressed by this PR Enforces truncation on properties parsed from user telemetry. ### Are there test cases added in this PR? _(If not, why?)_ Yes ### Checklists - [x] Added impacted package name to the issue description - [ ] Does this PR needs any fixes in the SDK Generator?** _(If so, create an Issue in the [Autorest/typescript](https://github.com/Azure/autorest.typescript) repository and link it here)_ - [x] Added a changelog (if necessary)
Azure · Sep 9, 2024 · 703c820 · 703c820
1 parent a637648
commit 703c820
Show file tree

Hide file tree

Showing 9 changed files with 145 additions and 8 deletions.
diff --git a/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md
@@ -4,7 +4,8 @@
 
 ### Other Changes
 
-- Updated OTel dependencies
+- Enforce property length limits on telemetry using truncation.
+- Updated OTel dependencies.
 
 ## 1.0.0-beta.25 (2024-08-14)
 

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/platform/nodejs/persist/fileSystemPersist.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/platform/nodejs/persist/fileSystemPersist.ts
@@ -99,7 +99,6 @@ export class FileSystemPersist implements PersistentStorage {
       try {
         const buffer = await this._getFirstFileOnDisk();
         if (buffer) {
-          // eslint-disable-next-line @typescript-eslint/no-unsafe-return
           return JSON.parse(buffer.toString("utf8"));
         }
       } catch (e: any) {

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/types.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/types.ts
@@ -71,3 +71,14 @@ export enum BreezePerformanceCounterNames {
   REQUEST_RATE = "\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec",
   REQUEST_DURATION = "\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Request Execution Time",
 }
+
+/**
+ * Property Max Lengths
+ * @internal
+ */
+export enum MaxPropertyLengths {
+  NINE_BIT = 512,
+  TEN_BIT = 1024,
+  THIRTEEN_BIT = 8192,
+  FIFTEEN_BIT = 32768,
+}
diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/logUtils.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/logUtils.ts
@@ -20,7 +20,7 @@ import {
   SEMATTRS_EXCEPTION_STACKTRACE,
   SEMATTRS_EXCEPTION_TYPE,
 } from "@opentelemetry/semantic-conventions";
-import { Measurements, Properties, Tags } from "../types";
+import { MaxPropertyLengths, Measurements, Properties, Tags } from "../types";
 import { diag } from "@opentelemetry/api";
 import {
   ApplicationInsightsAvailabilityBaseType,
@@ -92,6 +92,15 @@ export function logToEnvelope(log: ReadableLogRecord, ikey: string): Envelope |
       return;
     }
   }
+  // Truncate properties
+  if (baseData.message) {
+    baseData.message = String(baseData.message).substring(0, MaxPropertyLengths.FIFTEEN_BIT);
+  }
+  if (properties) {
+    for (const key of Object.keys(properties)) {
+      properties[key] = String(properties[key]).substring(0, MaxPropertyLengths.THIRTEEN_BIT);
+    }
+  }
   return {
     name,
     sampleRate,

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/spanUtils.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/spanUtils.ts
@@ -40,7 +40,7 @@ import {
   isSqlDB,
   serializeAttribute,
 } from "./common";
-import { Tags, Properties, MSLink, Measurements } from "../types";
+import { Tags, Properties, MSLink, Measurements, MaxPropertyLengths } from "../types";
 import { parseEventHubSpan } from "./eventhub";
 import { AzureMonitorSampleRate, DependencyTypes, MS_LINKS } from "./constants/applicationinsights";
 import { AzNamespace, MicrosoftEventHub } from "./constants/span/azAttributes";
@@ -345,6 +345,34 @@ export function readableSpanToEnvelope(span: ReadableSpan, ikey: string): Envelo
     }
   }
 
+  // Truncate properties
+  if (baseData.id) {
+    baseData.id = baseData.id.substring(0, MaxPropertyLengths.NINE_BIT);
+  }
+  if (baseData.name) {
+    baseData.name = baseData.name.substring(0, MaxPropertyLengths.TEN_BIT);
+  }
+  if (baseData.resultCode) {
+    baseData.resultCode = String(baseData.resultCode).substring(0, MaxPropertyLengths.TEN_BIT);
+  }
+  if (baseData.data) {
+    baseData.data = String(baseData.data).substring(0, MaxPropertyLengths.THIRTEEN_BIT);
+  }
+  if (baseData.type) {
+    baseData.type = String(baseData.type).substring(0, MaxPropertyLengths.TEN_BIT);
+  }
+  if (baseData.target) {
+    baseData.target = String(baseData.target).substring(0, MaxPropertyLengths.TEN_BIT);
+  }
+  if (baseData.properties) {
+    for (const key of Object.keys(baseData.properties)) {
+      baseData.properties[key] = baseData.properties[key].substring(
+        0,
+        MaxPropertyLengths.THIRTEEN_BIT,
+      );
+    }
+  }
+
   return {
     name,
     sampleRate,
@@ -438,6 +466,18 @@ export function spanEventsToEnvelopes(span: ReadableSpan, ikey: string): Envelop
       if (span.attributes[AzureMonitorSampleRate]) {
         sampleRate = Number(span.attributes[AzureMonitorSampleRate]);
       }
+      // Truncate properties
+      if (baseData.message) {
+        baseData.message = String(baseData.message).substring(0, MaxPropertyLengths.FIFTEEN_BIT);
+      }
+      if (baseData.properties) {
+        for (const key of Object.keys(baseData.properties)) {
+          baseData.properties[key] = baseData.properties[key].substring(
+            0,
+            MaxPropertyLengths.THIRTEEN_BIT,
+          );
+        }
+      }
       const env: Envelope = {
         name: name,
         time: time,

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/fileSystemPersist.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/fileSystemPersist.test.ts
@@ -69,7 +69,6 @@ describe("FileSystemPersist", () => {
   afterEach((done) => {
     fs.readdir(tempDir, (err, files) => {
       if (err) {
-        // eslint-disable-next-line no-console
         console.error(err);
         done();
       } else {

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/logUtils.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/logUtils.test.ts
@@ -7,7 +7,7 @@ import {
   SemanticResourceAttributes,
 } from "@opentelemetry/semantic-conventions";
 
-import { Tags, Properties, Measurements } from "../../src/types";
+import { Tags, Properties, Measurements, MaxPropertyLengths } from "../../src/types";
 import { getInstance } from "../../src/platform";
 import {
   AvailabilityData,
@@ -217,6 +217,49 @@ describe("logUtils.ts", () => {
       );
     });
 
+    it("should truncate properties of a Message Envelope", () => {
+      const data: MessageData = {
+        message: "a".repeat(MaxPropertyLengths.FIFTEEN_BIT + 1),
+        severityLevel: "Verbose",
+        measurements: { testMeasurement: 1 },
+        version: 2,
+      };
+      testLogRecord.attributes = {
+        "_MS.baseType": "MessageData",
+        "extra.attribute": "foo",
+        [SemanticAttributes.MESSAGE_TYPE]: "test message type",
+      };
+      testLogRecord.body = data;
+
+      const expectedTime = hrTimeToDate(testLogRecord.hrTime);
+      const expectedProperties = {
+        "extra.attribute": "foo",
+        [SemanticAttributes.MESSAGE_TYPE]: "test message type",
+      };
+      const expectedMeasurements: Measurements = {
+        testMeasurement: 1,
+      };
+      const expectedBaseData: Partial<MessageData> = {
+        message: "a".repeat(MaxPropertyLengths.FIFTEEN_BIT),
+        severityLevel: `Verbose`,
+        version: 2,
+        properties: expectedProperties,
+        measurements: expectedMeasurements,
+      };
+
+      const envelope = logToEnvelope(testLogRecord as ReadableLogRecord, "ikey");
+      assertEnvelope(
+        envelope,
+        "Microsoft.ApplicationInsights.Message",
+        100,
+        "MessageData",
+        expectedProperties,
+        expectedMeasurements,
+        expectedBaseData,
+        expectedTime,
+      );
+    });
+
     it("should create a Exception Envelope", () => {
       const data: TelemetryExceptionData = {
         message: "testMessage",

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/spanUtils.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/spanUtils.test.ts
@@ -32,7 +32,7 @@ import {
   SEMRESATTRS_SERVICE_NAMESPACE,
 } from "@opentelemetry/semantic-conventions";
 
-import { Tags, Properties, Measurements } from "../../src/types";
+import { Tags, Properties, Measurements, MaxPropertyLengths } from "../../src/types";
 import { Context, getInstance } from "../../src/platform";
 import { readableSpanToEnvelope, spanEventsToEnvelopes } from "../../src/utils/spanUtils";
 import {
@@ -1304,6 +1304,41 @@ describe("spanUtils.ts", () => {
       version: 2,
     };
 
+    assertEnvelope(
+      envelopes[0],
+      "Microsoft.ApplicationInsights.Message",
+      100,
+      "MessageData",
+      expectedTags,
+      expectedProperties,
+      undefined,
+      expectedBaseData,
+    );
+  });
+  it("should truncate message envelope for span events", () => {
+    const message = "a".repeat(MaxPropertyLengths.FIFTEEN_BIT + 1);
+    const span = new Span(
+      tracer,
+      ROOT_CONTEXT,
+      "parent span",
+      { traceId: "traceid", spanId: "spanId", traceFlags: 0 },
+      SpanKind.SERVER,
+      "parentSpanId",
+    );
+    span.addEvent(message);
+    span.end();
+    const envelopes = spanEventsToEnvelopes(span, "ikey");
+
+    const expectedTags: Tags = {};
+    expectedTags[KnownContextTagKeys.AiOperationId] = span.spanContext().traceId;
+    expectedTags[KnownContextTagKeys.AiOperationParentId] = "spanId";
+    const expectedProperties = {};
+    const expectedBaseData: Partial<MessageData> = {
+      message: message.substring(0, MaxPropertyLengths.FIFTEEN_BIT),
+      properties: expectedProperties,
+      version: 2,
+    };
+
     assertEnvelope(
       envelopes[0],
       "Microsoft.ApplicationInsights.Message",

diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/statsbeat.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/statsbeat.test.ts
@@ -101,7 +101,7 @@ describe("#AzureMonitorStatsbeatExporter", () => {
 
       it("should add correct network properites to the custom metric", (done) => {
         const statsbeat = new NetworkStatsbeatMetrics(options);
-        // eslint-disable-next-line no-unused-expressions, @typescript-eslint/no-unused-expressions
+        // eslint-disable-next-line no-unused-expressions
         statsbeat["statsCollectionShortInterval"];
         statsbeat.countSuccess(100);
         const metric = statsbeat["networkStatsbeatCollection"][0];