From cf15555bbe62a1e70e02a0c1abc12b9d2032fa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ida=20=C5=A0tambuk?= Date: Mon, 17 Jun 2024 11:47:56 +0200 Subject: [PATCH] Backend migration: Send all queries to backend if openSearchBackendFlow toggle is enabled (#409) * Send all queries to backend if feature toggle is enabled * Prepare 2.16.1 --- CHANGELOG.md | 4 + package.json | 2 +- src/datasource.test.ts | 562 +++++++---------------------------------- src/datasource.ts | 17 +- 4 files changed, 103 insertions(+), 482 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc0191aa..74fd8486 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## 2.16.1 + +- Send all queries to backend if feature toggle is enabled in [#409](https://github.com/grafana/opensearch-datasource/pull/409) + ## 2.16.0 - Bugfix: Pass docvalue_fields for elasticsearch in the backend flow in [#404](https://github.com/grafana/opensearch-datasource/pull/404) diff --git a/package.json b/package.json index f601a63a..4297731d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grafana-opensearch-datasource", - "version": "2.16.0", + "version": "2.16.1", "description": "", "scripts": { "build": "webpack -c ./.config/webpack/webpack.config.ts --env production", diff --git a/src/datasource.test.ts b/src/datasource.test.ts index 770787bd..2891871a 100644 --- a/src/datasource.test.ts +++ b/src/datasource.test.ts @@ -20,7 +20,7 @@ import { enhanceDataFrame, OpenSearchDatasource } from './datasource'; import { PPLFormatType } from './components/QueryEditor/PPLFormatEditor/formats'; // import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ // @ts-ignore -import { getBackendSrv, getTemplateSrv, DataSourceWithBackend } from '@grafana/runtime'; +import { getBackendSrv, getTemplateSrv, DataSourceWithBackend, config } from '@grafana/runtime'; import { Flavor, LuceneQueryType, @@ -62,9 +62,9 @@ jest.mock('@grafana/runtime', () => ({ }; }, getTemplateSrv: () => ({ - replace: jest.fn((text: string) => { + replace: jest.fn((text: string | undefined) => { // Replace all $ words, except global variables ($__interval, $__interval_ms, etc.) - they get interpolated on the BE - const resolved = text.replace(/\$(?!__)\w+/g, "resolvedVariable"); + const resolved = text?.replace(/\$(?!__)\w+/g, 'resolvedVariable') ?? ''; return resolved; }), getAdhocFilters: getAdHocFiltersMock, @@ -1123,7 +1123,6 @@ describe('OpenSearchDatasource', function (this: any) { expect(typeof JSON.parse(query.split('\n')[1]).query.bool.filter[0].range['@time'].gte).toBe('number'); }); }); - describe('query migration to the backend', () => { beforeAll(() => { datasourceRequestMock.mockImplementation(() => { @@ -1134,11 +1133,9 @@ describe('OpenSearchDatasource', function (this: any) { }); }); }); - it('should send raw_data queries', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const rawDataQuery: OpenSearchQuery = { + const testQueries: Array = [ + { + testCaseName: 'Lucene raw_data', refId: 'A', metrics: [ { @@ -1151,27 +1148,9 @@ describe('OpenSearchDatasource', function (this: any) { }, }, ], - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [rawDataQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send raw_document queries', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const rawDataQuery: OpenSearchQuery = { + }, + { + testCaseName: 'Lucene raw_document', refId: 'A', metrics: [ { @@ -1184,286 +1163,30 @@ describe('OpenSearchDatasource', function (this: any) { }, }, ], - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [rawDataQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send trace span queries', () => { - const rawDataQuery: OpenSearchQuery = { + }, + { + testCaseName: 'Lucene trace spans', refId: 'A', queryType: QueryType.Lucene, luceneQueryType: LuceneQueryType.Traces, query: 'traceId:test', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [rawDataQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send trace list queries', () => { - const rawDataQuery: OpenSearchQuery = { + }, + { + testCaseName: 'Lucene trace list', refId: 'A', queryType: QueryType.Lucene, luceneQueryType: LuceneQueryType.Traces, query: '', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [rawDataQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send logs queries in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const logsQuery: OpenSearchQuery = { + }, + { + testCaseName: 'Lucene logs', refId: 'A', metrics: [{ type: 'logs', id: '1' }], query: 'foo="bar"', queryType: QueryType.Lucene, - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [logsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send metric max group by terms query in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const metricQuery: OpenSearchQuery = { - refId: 'A', - bucketAggs: [ - { - field: 'AvgTicketPrice', - id: '2', - settings: { - min_doc_count: '0', - order: 'desc', - orderBy: '_term', - size: '10', - }, - type: 'terms', - }, - ], - metrics: [ - { - field: 'AvgTicketPrice', - id: '1', - type: 'max', - }, - ], - query: '*', - queryType: QueryType.Lucene, - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [metricQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send metric average and derivative query in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const metricQuery: OpenSearchQuery = { - refId: 'A', - bucketAggs: [ - { - field: 'timestamp', - id: '2', - settings: { - interval: '1d', - }, - type: 'date_histogram', - }, - ], - metrics: [ - { - field: 'AvgTicketPrice', - id: '1', - type: 'avg', - }, - { - field: '1', - id: '3', - type: 'derivative', - }, - ], - query: '*', - queryType: QueryType.Lucene, - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [metricQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send PPL logs format queries in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { - refId: 'A', - queryType: QueryType.PPL, - format: 'logs', - query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send PPL table format queries in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { - refId: 'A', - queryType: QueryType.PPL, - format: 'table', - query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('should send PPL time series format queries in Explore', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { - refId: 'A', - queryType: QueryType.PPL, - format: 'time_series', - query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Explore, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).toHaveBeenCalled(); - }); - - it('does not send logs queries in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const logsQuery: OpenSearchQuery = { - refId: 'A', - metrics: [{ type: 'logs', id: '1' }], - query: 'foo="bar"', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [logsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); - }); - - it('does not send metric max group by terms in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const metricQuery: OpenSearchQuery = { + }, + { + testCaseName: 'Lucene metrics', refId: 'A', bucketAggs: [ { @@ -1487,190 +1210,84 @@ describe('OpenSearchDatasource', function (this: any) { ], query: '*', queryType: QueryType.Lucene, - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [metricQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); - }); - - it('does not send metric average and derivative query in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const metricQuery: OpenSearchQuery = { - refId: 'A', - bucketAggs: [ - { - field: 'timestamp', - id: '2', - settings: { - interval: '1d', - }, - type: 'date_histogram', - }, - ], - metrics: [ - { - field: 'AvgTicketPrice', - id: '1', - type: 'avg', - }, - { - field: '1', - id: '3', - type: 'derivative', - }, - ], - query: '*', - queryType: QueryType.Lucene, - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [metricQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); - }); - - it('does not send PPL logs format queries in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { + }, + { + testCaseName: 'PPL Logs', refId: 'A', queryType: QueryType.PPL, format: 'logs', query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); - }); + }, + { + testCaseName: 'PPL Table', - it('does not send PPL table format queries in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { refId: 'A', queryType: QueryType.PPL, format: 'table', query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); - }); - - it('does not send PPL time series format queries in Dashboard to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const pplLogsQuery: OpenSearchQuery = { + }, + { + testCaseName: 'PPL Time Series', refId: 'A', queryType: QueryType.PPL, format: 'time_series', query: 'source = test-index', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [pplLogsQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); + }, + ]; + testQueries.forEach((query) => { + it(`should send ${query.testCaseName} query to the backend in Explore`, () => { + const request: DataQueryRequest = { + requestId: '', + interval: '', + intervalMs: 1, + scopedVars: {}, + timezone: '', + app: CoreApp.Explore, + startTime: 0, + range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), + targets: [query], + }; + ctx.ds.query(request); + expect(mockedSuperQuery).toHaveBeenCalled(); + }); }); - - it('does not send query including types not yet migrated to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const rawDataQuery: OpenSearchQuery = { - refId: 'A', - query: '', - metrics: [ - { - id: '1', - type: 'raw_data', - settings: { - size: '500', - order: 'desc', - useTimeRange: true, - }, - }, - ], - bucketAggs: [], - }; - const countQuery: OpenSearchQuery = { - refId: 'A', - metrics: [{ type: 'count', id: '1' }], - query: 'foo="bar"', - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [rawDataQuery, countQuery], - }; - ctx.ds.query(request); - expect(mockedSuperQuery).not.toHaveBeenCalled(); - expect(datasourceRequestMock).toHaveBeenCalled(); + testQueries.forEach((query) => { + it(`should send ${query.testCaseName} query to the backend in Dashboards if openSearchBackendFlowEnabled feature toggle is enabled`, () => { + // @ts-ignore-next-line + config.featureToggles.openSearchBackendFlowEnabled = true; + const request: DataQueryRequest = { + requestId: '', + interval: '', + intervalMs: 1, + scopedVars: {}, + timezone: '', + app: CoreApp.Dashboard, + startTime: 0, + range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), + targets: [query], + }; + ctx.ds.query(request); + expect(mockedSuperQuery).toHaveBeenCalled(); + }); + it(`should't send ${query.testCaseName} query to the backend in Dashboards if openSearchBackendFlowEnabled feature toggle is disabled`, () => { + // @ts-ignore + config.featureToggles.openSearchBackendFlowEnabled = false; + const request: DataQueryRequest = { + requestId: '', + interval: '', + intervalMs: 1, + scopedVars: {}, + timezone: '', + app: CoreApp.Dashboard, + startTime: 0, + range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), + targets: [query], + }; + ctx.ds.query(request); + expect(mockedSuperQuery).not.toHaveBeenCalled(); + }); }); }); - describe('getSupportedQueryTypes', () => { it('should return Lucene when no other types are set', () => { const instanceSettings = { @@ -2069,7 +1686,14 @@ describe('OpenSearchDatasource', function (this: any) { it('should correctly interpolate variables in nested fields in Lucene query', () => { const query: OpenSearchQuery = { refId: 'A', - bucketAggs: [{field: 'avgPrice', settings:{interval: "$var", min_doc_count: "$var", trimEdges: "$var"}, type: 'date_histogram', id: '1'}], + bucketAggs: [ + { + field: 'avgPrice', + settings: { interval: '$var', min_doc_count: '$var', trimEdges: '$var' }, + type: 'date_histogram', + id: '1', + }, + ], metrics: [{ type: 'count', id: '1' }], query: '$var AND foo:bar', }; @@ -2079,8 +1703,8 @@ describe('OpenSearchDatasource', function (this: any) { expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.interval).toBe('resolvedVariable'); expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.min_doc_count).toBe('resolvedVariable'); expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.trimEdges).toBe('resolvedVariable'); - }) - + }); + it('correctly applies template variables and adhoc filters to Lucene queries', () => { const adHocFilters: AdHocVariableFilter[] = [ { key: 'bar', operator: '=', value: 'baz', condition: '' }, diff --git a/src/datasource.ts b/src/datasource.ts index 46f6a698..ef691d9f 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -28,6 +28,7 @@ import { DataSourceWithBackend, FetchError, TemplateSrv, + config, getBackendSrv, getDataSourceSrv, getTemplateSrv, @@ -531,18 +532,10 @@ export class OpenSearchDatasource extends DataSourceWithBackend): Observable { - // Gradually migrate queries to the backend in this condition - if (request.targets.every( - (target) => - target.metrics?.every( - (metric) => - metric.type === 'raw_data' || - metric.type === 'raw_document' || - (request.app === CoreApp.Explore && target.queryType === QueryType.Lucene) - ) || - (request.app === CoreApp.Explore && target.queryType === QueryType.PPL) || - target.luceneQueryType === LuceneQueryType.Traces - )) { + // @ts-ignore-next-line + const { openSearchBackendFlowEnabled } = config.featureToggles; + // Backend flow + if (request.app === CoreApp.Explore || openSearchBackendFlowEnabled) { return super.query(request).pipe( tap({ next: (response) => {