Skip to content

Commit

Permalink
feat(otlp-grpc-exporter-base): use statically generated protobuf code (
Browse files Browse the repository at this point in the history
…#3705)

Co-authored-by: Daniel Dyla <dyladan@users.noreply.github.com>
  • Loading branch information
pichlermarc and dyladan authored Apr 28, 2023
1 parent abfb1bb commit 2f1e316
Show file tree
Hide file tree
Showing 14 changed files with 246 additions and 72 deletions.
1 change: 1 addition & 0 deletions experimental/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ All notable changes to experimental packages in this project will be documented
* feat(otlp-transformer): support log records. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
* feat(otlp-grpc-exporter-base): support log records. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
* feat(exporter-logs-otlp-grpc): otlp-grpc exporter for logs. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
* feat(otlp-grpc-exporter-base): use statically generated protobuf code [#3705](https://github.com/open-telemetry/opentelemetry-js/pull/3705) @pichlermarc
* refactor(otlp-transformer): refine metric transformers. [#3770](https://github.com/open-telemetry/opentelemetry-js/pull/3770/) @llc1123

### :bug: (Bug Fix)
Expand Down
2 changes: 2 additions & 0 deletions experimental/packages/otlp-grpc-exporter-base/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
src/generated/*
!src/generated/.gitkeep
16 changes: 8 additions & 8 deletions experimental/packages/otlp-grpc-exporter-base/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,17 @@
"scripts": {
"prepublishOnly": "npm run compile",
"codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p ../../../",
"compile": "tsc --build",
"compile": "npm run protos && tsc --build",
"clean": "tsc --build --clean",
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"postcompile": "npm run submodule && npm run protos:copy",
"protos:copy": "cpx protos/opentelemetry/**/*.* build/protos/opentelemetry",
"protos": "npm run submodule && npm run protos:generate",
"protos:generate": "node ../../../scripts/generate-protos.js",
"submodule": "git submodule sync --recursive && git submodule update --init --recursive",
"tdd": "npm run test -- --watch-extensions ts --watch",
"test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'",
"version": "node ../../../scripts/version-update.js",
"watch": "npm run protos:copy && tsc -w",
"watch": "npm run protos && tsc -w",
"precompile": "lerna run version --scope $(npm pkg get name) --include-dependencies",
"prewatch": "npm run precompile"
},
Expand All @@ -40,7 +40,6 @@
"build/src/**/*.js",
"build/src/**/*.js.map",
"build/src/**/*.d.ts",
"build/protos/**/*.proto",
"doc",
"LICENSE",
"README.md"
Expand All @@ -64,16 +63,17 @@
"sinon": "15.0.0",
"ts-loader": "8.4.0",
"ts-mocha": "10.0.0",
"typescript": "4.4.4"
"typescript": "4.4.4",
"protobufjs-cli": "1.0.2"
},
"peerDependencies": {
"@opentelemetry/api": "^1.0.0"
},
"dependencies": {
"@grpc/grpc-js": "^1.7.1",
"@grpc/proto-loader": "^0.7.3",
"@opentelemetry/core": "1.12.0",
"@opentelemetry/otlp-exporter-base": "0.38.0"
"@opentelemetry/otlp-exporter-base": "0.38.0",
"protobufjs": "^7.2.2"
},
"homepage": "https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/otlp-grpc-exporter-base",
"sideEffects": false
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import * as root from './generated/root';
import * as grpc from '@grpc/grpc-js';
import { IExportLogsServiceRequest } from '@opentelemetry/otlp-transformer';
import { ExportType } from './internal-types';
import IExportLogsServiceResponse = root.opentelemetry.proto.collector.logs.v1.IExportLogsServiceResponse;

const responseType = root.opentelemetry.proto.collector.logs.v1
.ExportLogsServiceResponse as ExportType<IExportLogsServiceResponse>;

const requestType = root.opentelemetry.proto.collector.logs.v1
.ExportLogsServiceRequest as ExportType<IExportLogsServiceRequest>;

const logsServiceDefinition = {
export: {
path: '/opentelemetry.proto.collector.logs.v1.LogsService/Export',
requestStream: false,
responseStream: false,
requestSerialize: (arg: IExportLogsServiceRequest) => {
return Buffer.from(requestType.encode(arg).finish());
},
requestDeserialize: (arg: Buffer) => {
return requestType.decode(arg);
},
responseSerialize: (arg: IExportLogsServiceResponse) => {
return Buffer.from(responseType.encode(arg).finish());
},
responseDeserialize: (arg: Buffer) => {
return responseType.decode(arg);
},
},
};

// Creates a new instance of a gRPC service client for OTLP logs
export const LogsExportServiceClient: grpc.ServiceClientConstructor =
grpc.makeGenericClientConstructor(logsServiceDefinition, 'LogsExportService');
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import * as root from './generated/root';
import * as grpc from '@grpc/grpc-js';
import { IExportMetricsServiceRequest } from '@opentelemetry/otlp-transformer';
import { ExportType } from './internal-types';
import IExportMetricsServiceResponse = root.opentelemetry.proto.collector.metrics.v1.IExportMetricsServiceResponse;

const responseType = root.opentelemetry.proto.collector.metrics.v1
.ExportMetricsServiceResponse as ExportType<IExportMetricsServiceResponse>;

const requestType = root.opentelemetry.proto.collector.metrics.v1
.ExportMetricsServiceRequest as ExportType<IExportMetricsServiceRequest>;

const metricsServiceDefinition = {
export: {
path: '/opentelemetry.proto.collector.metrics.v1.MetricsService/Export',
requestStream: false,
responseStream: false,
requestSerialize: (arg: IExportMetricsServiceRequest) => {
return Buffer.from(requestType.encode(arg).finish());
},
requestDeserialize: (arg: Buffer) => {
return requestType.decode(arg);
},
responseSerialize: (arg: IExportMetricsServiceResponse) => {
return Buffer.from(responseType.encode(arg).finish());
},
responseDeserialize: (arg: Buffer) => {
return responseType.decode(arg);
},
},
};

// Creates a new instance of a gRPC service client for OTLP metrics
export const MetricExportServiceClient: grpc.ServiceClientConstructor =
grpc.makeGenericClientConstructor(
metricsServiceDefinition,
'MetricsExportService'
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import * as root from './generated/root';
import * as grpc from '@grpc/grpc-js';
import { IExportTraceServiceRequest } from '@opentelemetry/otlp-transformer';
import { ExportType } from './internal-types';
import IExportTraceServiceResponse = root.opentelemetry.proto.collector.trace.v1.IExportTraceServiceResponse;

const responseType = root.opentelemetry.proto.collector.trace.v1
.ExportTraceServiceResponse as ExportType<IExportTraceServiceResponse>;

const requestType = root.opentelemetry.proto.collector.trace.v1
.ExportTraceServiceRequest as ExportType<IExportTraceServiceRequest>;

const traceServiceDefinition = {
export: {
path: '/opentelemetry.proto.collector.trace.v1.TraceService/Export',
requestStream: false,
responseStream: false,
requestSerialize: (arg: IExportTraceServiceRequest) => {
return Buffer.from(requestType.encode(arg).finish());
},
requestDeserialize: (arg: Buffer) => {
return requestType.decode(arg);
},
responseSerialize: (arg: IExportTraceServiceResponse) => {
return Buffer.from(responseType.encode(arg).finish());
},
responseDeserialize: (arg: Buffer) => {
return responseType.decode(arg);
},
},
};

// Creates a new instance of a gRPC service client for exporting OTLP traces
export const TraceExportServiceClient: grpc.ServiceClientConstructor =
grpc.makeGenericClientConstructor(
traceServiceDefinition,
'TraceExportService'
);
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import type * as protobuf from 'protobufjs';

export interface ExportType<T, R = T & { toJSON: () => unknown }> {
encode(message: T, writer?: protobuf.Writer): protobuf.Writer;
decode(reader: protobuf.Reader | Uint8Array, length?: number): R;
}
4 changes: 3 additions & 1 deletion experimental/packages/otlp-grpc-exporter-base/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@ export interface GRPCQueueItem<ExportedItem> {
/**
* Service Client for sending spans/metrics/logs
*/
export interface ServiceClient extends grpc.Client {
export interface ServiceClient {
export: (
request: any,
metadata: grpc.Metadata,
options: grpc.CallOptions,
callback: Function
) => {};

close(): void;
}

/**
Expand Down
89 changes: 36 additions & 53 deletions experimental/packages/otlp-grpc-exporter-base/src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/

import * as grpc from '@grpc/grpc-js';
import * as protoLoader from '@grpc/proto-loader';
import { diag } from '@opentelemetry/api';
import { getEnv, globalErrorHandler } from '@opentelemetry/core';
import * as path from 'path';
Expand All @@ -28,11 +27,15 @@ import {
ServiceClientType,
} from './types';
import {
CompressionAlgorithm,
ExportServiceError,
OTLPExporterError,
CompressionAlgorithm,
} from '@opentelemetry/otlp-exporter-base';

import { MetricExportServiceClient } from './MetricsExportServiceClient';
import { TraceExportServiceClient } from './TraceExportServiceClient';
import { LogsExportServiceClient } from './LogsExportServiceClient';

export const DEFAULT_COLLECTOR_URL = 'http://localhost:4317';

export function onInit<ExportItem, ServiceRequest>(
Expand All @@ -46,61 +49,41 @@ export function onInit<ExportItem, ServiceRequest>(
collector.getUrlFromConfig(config)
);

const includeDirs = [path.resolve(__dirname, '..', 'protos')];
try {
if (collector.getServiceClientType() === ServiceClientType.SPANS) {
const client = new TraceExportServiceClient(collector.url, credentials, {
'grpc.default_compression_algorithm': collector.compression.valueOf(),
});

protoLoader
.load(collector.getServiceProtoPath(), {
keepCase: false,
longs: String,
enums: String,
defaults: true,
oneofs: true,
includeDirs,
})
.then(packageDefinition => {
const packageObject: any = grpc.loadPackageDefinition(packageDefinition);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
collector.serviceClient = client;
} else if (collector.getServiceClientType() === ServiceClientType.METRICS) {
const client = new MetricExportServiceClient(collector.url, credentials, {
'grpc.default_compression_algorithm': collector.compression.valueOf(),
});

const options = {
'grpc.default_compression_algorithm': collector.compression,
};

switch (collector.getServiceClientType()) {
case ServiceClientType.SPANS:
collector.serviceClient =
new packageObject.opentelemetry.proto.collector.trace.v1.TraceService(
collector.url,
credentials,
options
);
break;
case ServiceClientType.METRICS:
collector.serviceClient =
new packageObject.opentelemetry.proto.collector.metrics.v1.MetricsService(
collector.url,
credentials,
options
);
break;
case ServiceClientType.LOGS:
collector.serviceClient =
new packageObject.opentelemetry.proto.collector.logs.v1.LogsService(
collector.url,
credentials,
options
);
break;
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
collector.serviceClient = client;
} else if (collector.getServiceClientType() === ServiceClientType.LOGS) {
const client = new LogsExportServiceClient(collector.url, credentials, {
'grpc.default_compression_algorithm': collector.compression.valueOf(),
});

if (collector.grpcQueue.length > 0) {
const queue = collector.grpcQueue.splice(0);
queue.forEach((item: GRPCQueueItem<ExportItem>) => {
collector.send(item.objects, item.onSuccess, item.onError);
});
}
})
.catch(err => {
globalErrorHandler(err);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
collector.serviceClient = client;
}
} catch (err) {
globalErrorHandler(err);
}
if (collector.grpcQueue.length > 0) {
const queue = collector.grpcQueue.splice(0);
queue.forEach((item: GRPCQueueItem<ExportItem>) => {
collector.send(item.objects, item.onSuccess, item.onError);
});
}
}

export function send<ExportItem, ServiceRequest>(
Expand Down
4 changes: 3 additions & 1 deletion experimental/packages/otlp-grpc-exporter-base/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"allowJs": true,
"outDir": "build",
"rootDir": "."
},
"include": [
"src/**/*.ts",
"src/generated/*.js",
"src/generated/**/*.js",
"src/generated/**/*.ts",
"test/**/*.ts"
],
"references": [
Expand Down
Loading

0 comments on commit 2f1e316

Please sign in to comment.