diff --git a/api/.env.sample b/api/.env.sample index 5d1a9bb24..43dccbd00 100644 --- a/api/.env.sample +++ b/api/.env.sample @@ -3,6 +3,7 @@ SERVER_PORT=31200 CORS_ALLOW_ORIGIN= SECRET_KEY="secret key for encrypting datasource passwords" ENABLE_AUTH=0 +ENABLE_QUERY_PARSER=0 SUPER_ADMIN_PASSWORD= DATABASE_CONNECTION_TIMEOUT_MS= DATABASE_POOL_SIZE= diff --git a/api/README.md b/api/README.md index 4f5f27ebb..5ff8a5f3e 100644 --- a/api/README.md +++ b/api/README.md @@ -11,6 +11,7 @@ Add a `.env` file based on `.env.sample` - `CORS_ALLOW_ORIGIN` for configuring cors. separate multiple origins by `;`. Defaults to `http://localhost` - `SECRET_KEY` for encrypting and decrypting passwords used in datasource configurations - `ENABLE_AUTH` Whether to add authentication and authorization to routes. 0 = disabled, 1 = enabled +- `ENABLE_QUERY_PARSER` Whether to enable Server-Side Query parsing. 0 = disabled, 1 = enabled - `SUPER_ADMIN_PASSWORD` The password which will be configured for the superadmin account during migration. Must be configured before migration is run. If value is not set, password will be 'secret' - `DATABASE_CONNECTION_TIMEOUT_MS` for configuration the time after which the db connection will timeout in milliseconds. Default is 30000ms (30 seconds) - `DATABASE_POOL_SIZE` for configuration the maximum number of clients in the pool diff --git a/api/src/api_models/dashboard_content.ts b/api/src/api_models/dashboard_content.ts index 12f37ad94..097105d18 100644 --- a/api/src/api_models/dashboard_content.ts +++ b/api/src/api_models/dashboard_content.ts @@ -1,8 +1,118 @@ import { ApiModel, ApiModelProperty, SwaggerDefinitionConstant } from 'swagger-express-ts'; -import { IsObject, Length, IsString, IsOptional, ValidateNested, IsUUID, IsIn } from 'class-validator'; +import { IsObject, Length, IsString, IsOptional, ValidateNested, IsUUID, IsIn, IsArray } from 'class-validator'; import { Type } from 'class-transformer'; import { Authentication, FilterObject, PaginationRequest, PaginationResponse, SortRequest } from './base'; +@ApiModel({ + description: 'Definition Query object', + name: 'Query', +}) +export class Query { + @IsString() + @ApiModelProperty({ + description: 'Query ID', + required: true, + }) + id: string; + + @IsIn(['postgresql', 'mysql', 'http']) + @ApiModelProperty({ + description: 'Datasource type', + required: true, + }) + type: 'postgresql' | 'mysql' | 'http'; + + @IsString() + @ApiModelProperty({ + description: 'Datasource key', + required: true, + }) + key: string; + + @IsString() + @ApiModelProperty({ + description: 'Query SQL', + required: true, + }) + sql: string; + + @IsString() + @ApiModelProperty({ + description: 'Query pre-processing', + required: true, + }) + pre_process: string; +} + +@ApiModel({ + description: 'Definition SQL Snippets object', + name: 'Snippet', +}) +export class Snippet { + @IsString() + @ApiModelProperty({ + description: 'Snippet ID', + required: true, + }) + key: string; + + @IsString() + @ApiModelProperty({ + description: 'Snippet definition', + required: true, + }) + value: string; +} + +@ApiModel({ + description: 'Content definition object', + name: 'ContentDefinition', +}) +export class ContentDefinition { + @IsArray() + @Type(() => Query) + @ValidateNested({ each: true }) + @ApiModelProperty({ + description: 'Content query definitions', + required: true, + model: 'Query', + }) + queries: Query[]; + + @IsArray() + @Type(() => Snippet) + @ValidateNested({ each: true }) + @ApiModelProperty({ + description: 'Content sql snippet definitions', + required: true, + model: 'Snippet', + }) + sqlSnippets: Snippet[]; +} + +@ApiModel({ + description: 'Content object', + name: 'Content', +}) +export class Content { + @IsObject() + @Type(() => ContentDefinition) + @ValidateNested({ each: true }) + @ApiModelProperty({ + description: 'Content definitions', + required: true, + model: 'ContentDefinition', + }) + definition: ContentDefinition; + + @IsString() + @ApiModelProperty({ + description: 'Content schema version', + required: true, + }) + version: string; +} + @ApiModel({ description: 'Dashboard content entity', name: 'DashboardContent', @@ -23,11 +133,12 @@ export class DashboardContent { }) name: string; + @Type(() => Content) @ApiModelProperty({ description: 'content of the dashboard stored in json object format', - type: SwaggerDefinitionConstant.JSON, + model: 'Content', }) - content: object | null; + content: Content; @ApiModelProperty({ description: 'Create time', @@ -176,12 +287,14 @@ export class DashboardContentCreateRequest { name: string; @IsObject() + @Type(() => Content) + @ValidateNested({ each: true }) @ApiModelProperty({ description: 'content stored in json object format', required: true, - type: SwaggerDefinitionConstant.JSON, + model: 'Content', }) - content: Record; + content: Content; @IsOptional() @Type(() => Authentication) @@ -217,12 +330,14 @@ export class DashboardContentUpdateRequest { @IsOptional() @IsObject() + @Type(() => Content) + @ValidateNested({ each: true }) @ApiModelProperty({ description: 'content of the dashboard stored in json object format', required: false, - type: SwaggerDefinitionConstant.JSON, + model: 'Content', }) - content?: Record; + content?: Content; @IsOptional() @Type(() => Authentication) diff --git a/api/src/api_models/index.ts b/api/src/api_models/index.ts index 201c682be..3a05836f8 100644 --- a/api/src/api_models/index.ts +++ b/api/src/api_models/index.ts @@ -45,7 +45,7 @@ import { ApiKeyIDRequest, } from './api'; import { Role, RolePermission, RoleCreateOrUpdateRequest, RoleIDRequest } from './role'; -import { QueryRequest, HttpParams } from './query'; +import { QueryParams, QueryRequest, HttpParams, QueryStructureRequest } from './query'; import { Job, JobFilterObject, JobListRequest, JobPaginationResponse, JobSortObject, JobRunRequest } from './job'; import { Config, ConfigDescription, ConfigGetRequest, ConfigUpdateRequest } from './config'; import { @@ -74,6 +74,10 @@ import { DashboardContentSortObject, DashboardContentIDRequest, DashboardContentUpdateRequest, + Content, + ContentDefinition, + Query, + Snippet, } from './dashboard_content'; import { DashboardContentChangelog, @@ -154,8 +158,10 @@ export default { RoleCreateOrUpdateRequest, RoleIDRequest, + QueryParams, QueryRequest, HttpParams, + QueryStructureRequest, Job, JobFilterObject, @@ -192,6 +198,10 @@ export default { DashboardContentSortObject, DashboardContentIDRequest, DashboardContentUpdateRequest, + Content, + ContentDefinition, + Query, + Snippet, DashboardContentChangelog, DashboardContentChangelogFilterObject, diff --git a/api/src/api_models/query.ts b/api/src/api_models/query.ts index b453b8c2e..bf0a62f44 100644 --- a/api/src/api_models/query.ts +++ b/api/src/api_models/query.ts @@ -1,8 +1,30 @@ import { Type } from 'class-transformer'; -import { IsBoolean, IsIn, IsObject, IsOptional, IsString, ValidateNested } from 'class-validator'; +import { IsBoolean, IsIn, IsNumber, IsObject, IsOptional, IsString, ValidateNested } from 'class-validator'; import { ApiModel, ApiModelProperty, SwaggerDefinitionConstant } from 'swagger-express-ts'; import { Authentication } from './base'; +@ApiModel({ + description: 'Query params object', + name: 'QueryParams', +}) +export class QueryParams { + @IsObject() + @ApiModelProperty({ + description: 'Query filter params', + required: true, + type: SwaggerDefinitionConstant.JSON, + }) + filters: Record; + + @IsObject() + @ApiModelProperty({ + description: 'Query context params', + required: true, + type: SwaggerDefinitionConstant.JSON, + }) + context: Record; +} + @ApiModel({ description: 'Query object', name: 'QueryRequest', @@ -31,6 +53,30 @@ export class QueryRequest { }) query: string; + @IsString() + @ApiModelProperty({ + description: 'id of the dashboard content', + required: true, + }) + content_id: string; + + @IsString() + @ApiModelProperty({ + description: 'id of the query defined in dashboard content', + required: true, + }) + query_id: string; + + @IsObject() + @Type(() => QueryParams) + @ValidateNested({ each: true }) + @ApiModelProperty({ + description: 'Query params', + required: true, + model: 'QueryParams', + }) + params: QueryParams; + @IsOptional() @IsObject() @ApiModelProperty({ @@ -109,3 +155,67 @@ export class HttpParams { }) url: string; } + +@ApiModel({ + description: 'Query Structure object', + name: 'QueryStructureRequest', +}) +export class QueryStructureRequest { + @IsIn(['TABLES', 'COLUMNS', 'DATA', 'INDEXES', 'COUNT']) + @ApiModelProperty({ + description: `type of query. + TABLES = get all tables in database + COLUMNS = get column structure of table + DATA = get data of table + INDEXES = get indexes of table + COUNT = get total number of rows in table`, + required: true, + enum: ['TABLES', 'COLUMNS', 'DATA', 'INDEXES', 'COUNT'], + }) + query_type: 'TABLES' | 'COLUMNS' | 'DATA' | 'INDEXES' | 'COUNT'; + + @IsIn(['postgresql', 'mysql']) + @ApiModelProperty({ + description: 'datasource type of query', + required: true, + enum: ['postgresql', 'mysql'], + }) + type: 'postgresql' | 'mysql'; + + @IsString() + @ApiModelProperty({ + description: 'datasource key', + required: true, + }) + key: string; + + @IsString() + @ApiModelProperty({ + description: 'table schema', + required: true, + }) + table_schema: string; + + @IsString() + @ApiModelProperty({ + description: 'table schema', + required: true, + }) + table_name: string; + + @IsNumber() + @IsOptional() + @ApiModelProperty({ + description: 'Limit of query results. Default = 20', + required: false, + }) + limit?: number; + + @IsNumber() + @IsOptional() + @ApiModelProperty({ + description: 'Offset of query results, Default = 0', + required: false, + }) + offset?: number; +} diff --git a/api/src/controller/query.controller.ts b/api/src/controller/query.controller.ts index d5b77469e..468235834 100644 --- a/api/src/controller/query.controller.ts +++ b/api/src/controller/query.controller.ts @@ -4,9 +4,11 @@ import { controller, httpPost, interfaces } from 'inversify-express-utils'; import { ApiOperationPost, ApiPath } from 'swagger-express-ts'; import { QueryService } from '../services/query.service'; import { validate } from '../middleware/validation'; -import { QueryRequest } from '../api_models/query'; +import { QueryRequest, QueryStructureRequest } from '../api_models/query'; import permission from '../middleware/permission'; import { PERMISSIONS } from '../services/role.service'; +import { ApiKey } from '../api_models/api'; +import { Account } from '../api_models/account'; @ApiPath({ path: '/query', @@ -33,8 +35,54 @@ export class QueryController implements interfaces.Controller { @httpPost('/', permission({ match: 'all', permissions: [PERMISSIONS.DASHBOARD_VIEW] }), validate(QueryRequest)) public async query(req: express.Request, res: express.Response, next: express.NextFunction): Promise { try { - const { type, key, query, env, refresh_cache } = req.body as QueryRequest; - const result = await this.queryService.query(type, key, query, env || {}, refresh_cache); + const auth: Account | ApiKey | undefined = req.body.auth; + const { type, key, query, content_id, query_id, params, env, refresh_cache } = req.body as QueryRequest; + const result = await this.queryService.query( + type, + key, + query, + content_id, + query_id, + params, + env || {}, + refresh_cache, + req.locale, + auth, + ); + res.json(result); + } catch (error) { + next(error); + } + } + + @ApiOperationPost({ + path: '/structure', + description: 'query structure of selected datasource', + parameters: { + body: { description: 'Query Structure object', required: true, model: 'QueryStructureRequest' }, + }, + responses: { + 200: { description: 'Query result' }, + 500: { description: 'ApiError', model: 'ApiError' }, + }, + }) + @httpPost( + '/structure', + permission({ match: 'all', permissions: [PERMISSIONS.DASHBOARD_MANAGE] }), + validate(QueryStructureRequest), + ) + public async queryStructure(req: express.Request, res: express.Response, next: express.NextFunction): Promise { + try { + const { query_type, type, key, table_schema, table_name, limit, offset } = req.body as QueryStructureRequest; + const result = await this.queryService.queryStructure( + query_type, + type, + key, + table_schema, + table_name, + limit, + offset, + ); res.json(result); } catch (error) { next(error); diff --git a/api/src/locales/en.json b/api/src/locales/en.json index 0755ddd4a..f77993bd6 100644 --- a/api/src/locales/en.json +++ b/api/src/locales/en.json @@ -20,6 +20,7 @@ "DATASOURCE_CONNECTION_TEST_FAILED": "Testing datasource connection failed", "DATASOURCE_TYPE_KEY_ALREADY_EXISTS": "A data source with that type and key already exists", "DATASOURCE_ONLY_HTTP_IS_EDITABLE": "Only HTTP data source is editable", + "DATASOURCE_MISSING": "Several datasources were not found", "AUTH_NOT_ENABLED": "Authentication system is not enabled", "AUTH_MUST_BEARER": "Must authenticate with bearer token", "APIKEY_NO_DELETE_PRESET": "Preset apikey can not be deleted", @@ -74,5 +75,7 @@ "ROLE_PERMISSION_CUSTOM_FUNCTION_MANAGE": "Allows creating, editing, deleting custom functions", "ROLE_PERMISSION_SQL_SNIPPET_VIEW": "Allows viewing sql snippets", "ROLE_PERMISSION_SQL_SNIPPET_MANAGE": "Allows creating, editing, deleting sql snippets", - "ROLE_PERMISSION_PRESET": "Allows modification of preset assets" + "ROLE_PERMISSION_PRESET": "Allows modification of preset assets", + "QUERY_ID_NOT_FOUND": "A Query with that ID does not exist", + "QUERY_PARSING_ERROR": "Error parsing query" } diff --git a/api/src/locales/zh.json b/api/src/locales/zh.json index cde24fbc7..9a4148b7e 100644 --- a/api/src/locales/zh.json +++ b/api/src/locales/zh.json @@ -20,6 +20,7 @@ "DATASOURCE_CONNECTION_TEST_FAILED": "数据源测试连接失败", "DATASOURCE_TYPE_KEY_ALREADY_EXISTS": "具有该类型和密钥的数据源已存在", "DATASOURCE_ONLY_HTTP_IS_EDITABLE": "只有HTTP数据源可以编辑", + "DATASOURCE_MISSING": "有一些数据源找不到", "AUTH_NOT_ENABLED": "未启用身份验证系统", "AUTH_MUST_BEARER": "必须使用Bearer进行身份验证", "APIKEY_NO_DELETE_PRESET": "无法删除预设apikey", @@ -74,5 +75,7 @@ "ROLE_PERMISSION_CUSTOM_FUNCTION_MANAGE": "允许创建、编辑、删除自定义功能", "ROLE_PERMISSION_SQL_SNIPPET_VIEW": "允许查看sql snippets", "ROLE_PERMISSION_SQL_SNIPPET_MANAGE": "允许创建、编辑、删除 sql snippets", - "ROLE_PERMISSION_PRESET": "允许修改预设资产" + "ROLE_PERMISSION_PRESET": "允许修改预设资产", + "QUERY_ID_NOT_FOUND": "找不到查询", + "QUERY_PARSING_ERROR": "查询解析错误" } diff --git a/api/src/models/dashboard_content.ts b/api/src/models/dashboard_content.ts index 395ba3160..6b9e88a58 100644 --- a/api/src/models/dashboard_content.ts +++ b/api/src/models/dashboard_content.ts @@ -1,5 +1,6 @@ import { Entity, Column } from 'typeorm'; import { BaseModel } from './base'; +import { Content } from '../api_models/dashboard_content'; @Entity() export default class DashboardContent extends BaseModel { @@ -20,5 +21,5 @@ export default class DashboardContent extends BaseModel { nullable: false, name: 'content', }) - content: Record; + content: Content; } diff --git a/api/src/preset/scripts/configure_dashboards.ts b/api/src/preset/scripts/configure_dashboards.ts index 1819c0b91..2b4213afe 100644 --- a/api/src/preset/scripts/configure_dashboards.ts +++ b/api/src/preset/scripts/configure_dashboards.ts @@ -14,6 +14,7 @@ import { DashboardContentChangelogService } from '../../services/dashboard_conte import { FIXED_ROLE_TYPES } from '../../services/role.service'; import { QueryRunner, Repository } from 'typeorm'; import { ensureDirSync } from 'fs-extra'; +import { Content } from '../../api_models/dashboard_content'; type Source = { type: string; @@ -41,7 +42,7 @@ async function upsert() { const errors: { [type: string]: string[] } = {}; for (let i = 0; i < files.length; i++) { - const config: Record = JSON.parse(readFileSync(path.join(basePath, files[i]), 'utf-8')); + const config: Content = JSON.parse(readFileSync(path.join(basePath, files[i]), 'utf-8')); await checkConfigForErrors(datasourceRepo, config, errors); const name = dashboardNames[i]; await upsertDashboard(queryRunner, name, config, superadmin.id); @@ -89,7 +90,7 @@ async function purgeRemovedDashboards(dashboardRepo: Repository, dash async function checkConfigForErrors( datasourceRepo: Repository, - config: Record, + config: Content, errors: { [type: string]: string[] }, ) { for (let i = 0; i < config.definition.queries.length; i++) { @@ -105,12 +106,7 @@ async function checkConfigForErrors( } } -async function upsertDashboard( - queryRunner: QueryRunner, - name: string, - config: Record, - superadminId: string, -) { +async function upsertDashboard(queryRunner: QueryRunner, name: string, config: Content, superadminId: string) { const dashboardChangelogRepo = queryRunner.manager.getRepository(DashboardChangelog); const dashboardRepo = queryRunner.manager.getRepository(Dashboard); const dashboardContentChangelogRepo = queryRunner.manager.getRepository(DashboardContentChangelog); diff --git a/api/src/services/dashboard_content.service.ts b/api/src/services/dashboard_content.service.ts index 938d2f841..03dd80e4d 100644 --- a/api/src/services/dashboard_content.service.ts +++ b/api/src/services/dashboard_content.service.ts @@ -1,6 +1,7 @@ import _, { has, omit } from 'lodash'; import { PaginationRequest } from '../api_models/base'; import { + Content, DashboardContentFilterObject, DashboardContentPaginationResponse, DashboardContentSortObject, @@ -21,9 +22,34 @@ import { HIDDEN_PERMISSIONS } from './role.service'; import DashboardPermission from '../models/dashboard_permission'; import { PermissionResource } from '../api_models/dashboard_permission'; import { injectable } from 'inversify'; +import DataSource from '../models/datasource'; +import { Any } from 'typeorm'; @injectable() export class DashboardContentService { + private async checkQueryDatasources(content: Content, locale: string): Promise { + const dataSourceRepo = dashboardDataSource.getRepository(DataSource); + const errors: { [type: string]: string[] } = {}; + const keys = new Set(content.definition.queries.map((x) => x.key)); + const sources = await dataSourceRepo.findBy({ key: Any(Array.from(keys)) }); + content.definition.queries.forEach((q) => { + if (!sources.find((s) => s.type === q.type && s.key === q.key)) { + if (!errors[q.type]) { + errors[q.type] = []; + } + if (!errors[q.type].includes(q.key)) { + errors[q.type].push(q.key); + } + } + }); + if (!_.isEmpty(errors)) { + throw new ApiError(BAD_REQUEST, { + message: translate('DATASOURCE_MISSING', locale), + missing: errors, + }); + } + } + async list( dashboard_id: string, filter: DashboardContentFilterObject | undefined, @@ -93,16 +119,12 @@ export class DashboardContentService { }; } - async create( - dashboard_id: string, - name: string, - content: Record, - locale: string, - ): Promise { + async create(dashboard_id: string, name: string, content: Content, locale: string): Promise { const dashboardContentRepo = dashboardDataSource.getRepository(DashboardContent); if (await dashboardContentRepo.exist({ where: { dashboard_id, name } })) { throw new ApiError(BAD_REQUEST, { message: translate('DASHBOARD_CONTENT_NAME_ALREADY_EXISTS', locale) }); } + await this.checkQueryDatasources(content, locale); const dashboardContent = new DashboardContent(); dashboardContent.dashboard_id = dashboard_id; dashboardContent.name = name; @@ -118,7 +140,7 @@ export class DashboardContentService { async update( id: string, name: string | undefined, - content: Record | undefined, + content: Content | undefined, locale: string, auth?: Account | ApiKey, ): Promise { @@ -157,6 +179,9 @@ export class DashboardContentService { throw new ApiError(BAD_REQUEST, { message: translate('DASHBOARD_CONTENT_NAME_ALREADY_EXISTS', locale) }); } } + if (content !== undefined) { + await this.checkQueryDatasources(content, locale); + } const originalDashboardContent = _.cloneDeep(dashboardContent); dashboardContent.name = name === undefined ? dashboardContent.name : name; dashboardContent.content = content === undefined ? dashboardContent.content : content; diff --git a/api/src/services/query.service.ts b/api/src/services/query.service.ts index 9f9f84986..97aa93fb0 100644 --- a/api/src/services/query.service.ts +++ b/api/src/services/query.service.ts @@ -1,13 +1,24 @@ import { APIClient } from '../utils/api_client'; import { DataSourceService } from './datasource.service'; -import { DataSource } from 'typeorm'; +import { Any, DataSource } from 'typeorm'; import { configureDatabaseSource } from '../utils/helpers'; import { validateClass } from '../middleware/validation'; -import { HttpParams } from '../api_models/query'; +import { HttpParams, QueryParams } from '../api_models/query'; import { sqlRewriter } from '../plugins'; -import { ApiError, QUERY_ERROR } from '../utils/errors'; +import { ApiError, BAD_REQUEST, QUERY_ERROR } from '../utils/errors'; import { getFsCache, getFsCacheKey, isFsCacheEnabled, putFsCache } from '../utils/fs_cache'; import { injectable } from 'inversify'; +import { ApiKey } from '../api_models/api'; +import { Account } from '../api_models/account'; +import { dashboardDataSource } from '../data_sources/dashboard'; +import DashboardContent from '../models/dashboard_content'; +import { DashboardPermissionService } from './dashboard_permission.service'; +import { has } from 'lodash'; +import { translate } from '../utils/i18n'; +import SqlSnippet from '../models/sql_snippet'; +import { QUERY_PARSING_ENABLED } from '../utils/constants'; +import { PERMISSIONS } from './role.service'; +import { Query, Snippet } from '../api_models/dashboard_content'; @injectable() export class QueryService { @@ -41,17 +52,181 @@ export class QueryService { } } - async query(type: string, key: string, query: string, env: Record, refresh_cache = false): Promise { - let q: string = query; - if (['postgresql', 'mysql'].includes(type)) { - const { error, sql } = await sqlRewriter(query, env); + private getDBStructureSql(): string { + return 'SELECT table_schema, table_name, table_type FROM information_schema.tables ORDER BY table_schema, table_name'; + } + + private getColumnStructureSql(type: string, table_schema: string, table_name: string): string { + if (type === 'postgresql') { + const attrelid = `'${table_schema}.${table_name}'::regclass`; + return ` + SELECT + ordinal_position, + UPPER(pc.contype) AS column_key, + pg_get_constraintdef(pc.oid) AS column_key_text, + column_name, + format_type(atttypid, atttypmod) AS column_type, + is_nullable, + column_default, + pg_catalog.col_description(${attrelid}, ordinal_position) AS column_comment + FROM + information_schema.columns + JOIN pg_attribute pa ON pa.attrelid = ${attrelid} + AND attname = column_name + LEFT JOIN pg_constraint pc ON pc.conrelid = ${attrelid} AND ordinal_position = any(pc.conkey) + WHERE + table_name = '${table_name}' AND table_schema = '${table_schema}'; + `; + } + if (type === 'mysql') { + return ` + SELECT ordinal_position, column_key, column_name, column_type, is_nullable, column_default, column_comment + FROM information_schema.columns + WHERE table_name = '${table_name}' AND table_schema = '${table_schema}' + `; + } + return ''; + } + + private getDataSql(table_schema: string, table_name: string, limit: number, offset: number): string { + return ` + SELECT * + FROM ${table_schema}.${table_name} + LIMIT ${limit} OFFSET ${offset}`; + } + + private getIndexesSql(type: string, table_schema: string, table_name: string): string { + if (type === 'postgresql') { + return ` + SELECT + ix.relname AS index_name, + upper(am.amname) AS index_algorithm, + indisunique AS is_unique, + pg_get_indexdef(indexrelid) AS index_definition, + CASE WHEN position(' WHERE ' IN pg_get_indexdef(indexrelid)) > 0 THEN + regexp_replace(pg_get_indexdef(indexrelid), '.+WHERE ', '') + WHEN position(' WITH ' IN pg_get_indexdef(indexrelid)) > 0 THEN + regexp_replace(pg_get_indexdef(indexrelid), '.+WITH ', '') + ELSE + '' + END AS condition, + pg_catalog.obj_description(i.indexrelid, 'pg_class') AS comment + FROM + pg_index i + JOIN pg_class t ON t.oid = i.indrelid + JOIN pg_class ix ON ix.oid = i.indexrelid + JOIN pg_namespace n ON t.relnamespace = n.oid + JOIN pg_am AS am ON ix.relam = am.oid + WHERE + t.relname = '${table_name}' AND n.nspname = '${table_schema}'; + `; + } + if (type === 'mysql') { + return ` + SELECT + sub_part AS index_length, + index_name AS index_name, + index_type AS index_algorithm, + CASE non_unique WHEN 0 THEN 'TRUE' ELSE 'FALSE' END AS is_unique, + column_name AS column_name + FROM + information_schema.statistics + WHERE + table_name = '${table_name}' AND table_schema = '${table_schema}' + ORDER BY + seq_in_index ASC; + `; + } + return ''; + } + + private getCountSql(table_schema: string, table_name: string): string { + return ` + SELECT count(*) AS total + FROM ${table_schema}.${table_name} + `; + } + + async queryStructure( + query_type: string, + type: string, + key: string, + table_schema: string, + table_name: string, + limit = 20, + offset = 0, + ): Promise { + let sql: string; + switch (query_type) { + case 'TABLES': + sql = this.getDBStructureSql(); + break; + case 'COLUMNS': + sql = this.getColumnStructureSql(type, table_schema, table_name); + break; + case 'DATA': + sql = this.getDataSql(table_schema, table_name, limit, offset); + break; + case 'INDEXES': + sql = this.getIndexesSql(type, table_schema, table_name); + break; + case 'COUNT': + sql = this.getCountSql(table_schema, table_name); + break; + + default: + return null; + } + + let result; + switch (type) { + case 'postgresql': + result = await this.postgresqlQuery(key, sql); + break; + + case 'mysql': + result = await this.mysqlQuery(key, sql); + break; + + default: + return null; + } + return result; + } + + async query( + type: string, + key: string, + query: string, + content_id: string, + query_id: string, + params: QueryParams, + env: Record, + refresh_cache = false, + locale: string, + auth?: Account | ApiKey, + ): Promise { + const { parsedType, parsedKey, parsedQuery } = await this.prepareQuery( + type, + key, + query, + content_id, + query_id, + params, + locale, + auth, + ); + + let q: string = parsedQuery; + if (['postgresql', 'mysql'].includes(parsedType)) { + const { error, sql } = await sqlRewriter(q, env); if (error) { throw new ApiError(QUERY_ERROR, { message: error }); } q = sql; } const fsCacheEnabled = await isFsCacheEnabled(); - const cacheKey = getFsCacheKey(`${type}:${key}:${q}`); + const cacheKey = getFsCacheKey(`${parsedType}:${parsedKey}:${q}`); if (fsCacheEnabled && !refresh_cache) { const cached = await getFsCache(cacheKey); if (cached) { @@ -59,17 +234,17 @@ export class QueryService { } } let result; - switch (type) { + switch (parsedType) { case 'postgresql': - result = await this.postgresqlQuery(key, q); + result = await this.postgresqlQuery(parsedKey, q); break; case 'mysql': - result = await this.mysqlQuery(key, q); + result = await this.mysqlQuery(parsedKey, q); break; case 'http': - result = await this.httpQuery(key, q); + result = await this.httpQuery(parsedKey, q); break; default: @@ -81,6 +256,125 @@ export class QueryService { return result; } + private async prepareQuery( + type: string, + key: string, + query: string, + content_id: string, + query_id: string, + params: QueryParams, + locale: string, + auth?: Account | ApiKey, + ): Promise<{ parsedType: string; parsedKey: string; parsedQuery: string }> { + if (!QUERY_PARSING_ENABLED || auth?.permissions.includes(PERMISSIONS.DASHBOARD_MANAGE)) { + return { parsedType: type, parsedKey: key, parsedQuery: query }; + } + const dashboardContent = await dashboardDataSource + .getRepository(DashboardContent) + .findOneByOrFail({ id: content_id }); + + let auth_id: string | undefined; + let auth_type: 'APIKEY' | 'ACCOUNT' | undefined; + let auth_role_id: string | undefined; + let auth_permissions: string[] | undefined; + if (auth) { + auth_id = auth.id; + auth_type = has(auth, 'app_id') ? 'APIKEY' : 'ACCOUNT'; + auth_role_id = auth.role_id; + auth_permissions = auth.permissions; + } + await DashboardPermissionService.checkPermission( + dashboardContent.dashboard_id, + 'VIEW', + locale, + auth_id, + auth_type, + auth_role_id, + auth_permissions, + ); + + const content = dashboardContent.content; + const rawQuery = content.definition.queries.find((x) => x.id === query_id); + if (!rawQuery) { + throw new ApiError(BAD_REQUEST, { message: translate('QUERY_ID_NOT_FOUND', locale) }); + } + if (rawQuery.type === 'http') { + return await this.prepareHTTPQuery(rawQuery, params, locale); + } + return await this.prepareDBQuery(rawQuery, content.definition.sqlSnippets, params, locale); + } + + private async prepareHTTPQuery( + rawQuery: Query, + params: QueryParams, + locale: string, + ): Promise<{ parsedType: string; parsedKey: string; parsedQuery: string }> { + try { + const queryFunc = new Function(`return ${rawQuery.pre_process}`)(); + const query = queryFunc(params); + return { parsedKey: rawQuery.key, parsedType: rawQuery.type, parsedQuery: JSON.stringify(query) }; + } catch (err) { + throw new ApiError(BAD_REQUEST, { message: translate('QUERY_PARSING_ERROR', locale), details: err.message }); + } + } + + private async prepareDBQuery( + rawQuery: Query, + snippets: Snippet[], + params: QueryParams, + locale: string, + ): Promise<{ parsedType: string; parsedKey: string; parsedQuery: string }> { + try { + const query = rawQuery.sql; + + const sqlSnippetKeys = this.extractKeysFromQuery(query, /sql_snippets\.[\w]+/gm, 'sql_snippets.', ''); + const sqlSnippets = + sqlSnippetKeys.size > 0 + ? snippets + .filter((el) => sqlSnippetKeys.has(el.key)) + .reduce((acc, cur) => { + acc[cur.key] = new Function(...Object.keys(params), `return \`${cur.value}\``)( + ...Object.values(params), + ); + return acc; + }, {}) + : {}; + + const globalSqlSnippetKeys = this.extractKeysFromQuery( + query, + /global_sql_snippets\.[\w]+/gm, + 'global_sql_snippets.', + '', + ); + const globalSqlSnippets = + globalSqlSnippetKeys.size > 0 + ? (await dashboardDataSource.getRepository(SqlSnippet).findBy({ id: Any([...globalSqlSnippetKeys]) })).reduce( + (acc, cur) => { + acc[cur.id] = new Function(...Object.keys(params), `return \`${cur.content}\``)( + ...Object.values(params), + ); + return acc; + }, + {}, + ) + : []; + + const sql = new Function(...Object.keys(params), 'sql_snippets', 'global_sql_snippets', `return \`${query}\``)( + ...Object.values(params), + sqlSnippets, + globalSqlSnippets, + ); + + return { parsedType: rawQuery.type, parsedKey: rawQuery.key, parsedQuery: sql }; + } catch (err) { + throw new ApiError(BAD_REQUEST, { message: translate('QUERY_PARSING_ERROR', locale), details: err.message }); + } + } + + private extractKeysFromQuery(query: string, regex: RegExp, search: string, replace: string): Set { + return new Set(query.match(regex)?.map((match) => match.replace(search, replace))); + } + private async postgresqlQuery(key: string, sql: string): Promise { let source = QueryService.getDBConnection('postgresql', key); if (!source) { diff --git a/api/src/utils/constants.ts b/api/src/utils/constants.ts index a6ecad4d9..dc2e6b20d 100644 --- a/api/src/utils/constants.ts +++ b/api/src/utils/constants.ts @@ -6,3 +6,4 @@ export const DATABASE_CONNECTION_TIMEOUT_MS = parseInt(process.env.DATABASE_CONN export const DATABASE_POOL_SIZE = parseInt(process.env.DATABASE_POOL_SIZE ?? '10'); export const DEFAULT_LANGUAGE = 'en'; export const FS_CACHE_RETAIN_TIME = '86400'; +export const QUERY_PARSING_ENABLED = process.env.ENABLE_QUERY_PARSER === '1'; diff --git a/api/src/utils/i18n.ts b/api/src/utils/i18n.ts index aff2b6bc4..b9a961877 100644 --- a/api/src/utils/i18n.ts +++ b/api/src/utils/i18n.ts @@ -31,7 +31,8 @@ type DATASOURCE_KEYS = | 'DATASOURCE_NO_DELETE_PRESET' | 'DATASOURCE_CONNECTION_TEST_FAILED' | 'DATASOURCE_TYPE_KEY_ALREADY_EXISTS' - | 'DATASOURCE_ONLY_HTTP_IS_EDITABLE'; + | 'DATASOURCE_ONLY_HTTP_IS_EDITABLE' + | 'DATASOURCE_MISSING'; type AUTH_KEYS = 'AUTH_NOT_ENABLED' | 'AUTH_MUST_BEARER'; @@ -96,6 +97,8 @@ export type ROLE_PERMISSION_KEYS = | 'ROLE_PERMISSION_SQL_SNIPPET_MANAGE' | 'ROLE_PERMISSION_PRESET'; +type QUERY_KEYS = 'QUERY_ID_NOT_FOUND' | 'QUERY_PARSING_ERROR'; + type LANGUAGE_KEYS = | ACCOUNT_KEYS | DATASOURCE_KEYS @@ -110,7 +113,8 @@ type LANGUAGE_KEYS = | CUSTOM_FUNCTION_KEYS | ROLE_KEYS | SQL_SNIPPET_KEYS - | ROLE_PERMISSION_KEYS; + | ROLE_PERMISSION_KEYS + | QUERY_KEYS; export function translate(key: LANGUAGE_KEYS, locale: string): string { return i18n.__({ phrase: key, locale }); diff --git a/api/tests/e2e/06_query.test.ts b/api/tests/e2e/06_query.test.ts index f3795def5..78ef08675 100644 --- a/api/tests/e2e/06_query.test.ts +++ b/api/tests/e2e/06_query.test.ts @@ -1,13 +1,18 @@ import { connectionHook } from './jest.util'; -import { HttpParams, QueryRequest } from '~/api_models/query'; +import { QueryRequest, QueryStructureRequest } from '~/api_models/query'; import { app } from '~/server'; import request from 'supertest'; import { AccountLoginRequest, AccountLoginResponse } from '~/api_models/account'; -import { FIXED_ROLE_PERMISSIONS, FIXED_ROLE_TYPES } from '~/services/role.service'; +import { DashboardCreateRequest } from '~/api_models/dashboard'; +import { DashboardContentCreateRequest } from '~/api_models/dashboard_content'; +import { dashboardDataSource } from '~/data_sources/dashboard'; +import Dashboard from '~/models/dashboard'; describe('QueryController', () => { connectionHook(); let superadminLogin: AccountLoginResponse; + let dashboardId: string; + let dashboardContentId: string; const server = request(app); beforeAll(async () => { @@ -19,6 +24,89 @@ describe('QueryController', () => { const response = await server.post('/account/login').send(query); superadminLogin = response.body; + const queryDashboardRequest: DashboardCreateRequest = { + name: 'queryDashboard', + group: '', + }; + + const queryDashboardResponse = await server + .post('/dashboard/create') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(queryDashboardRequest); + + dashboardId = queryDashboardResponse.body.id; + + const queryDashboardContentRequest: DashboardContentCreateRequest = { + dashboard_id: dashboardId, + name: 'queryDashboardContent', + content: { + version: '', + definition: { + queries: [ + { + id: 'pgQuery', + type: 'postgresql', + key: 'preset', + sql: 'SELECT ${sql_snippets.role_columns} FROM role WHERE id = ${filters.role_id} AND ${context.true}', + pre_process: '', + }, + { + id: 'httpGetQuery', + type: 'http', + key: 'jsonplaceholder_renamed', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = {};\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "GET",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpPostQuery', + type: 'http', + key: 'jsonplaceholder_renamed', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = { "title": "foo", "body": "bar", "userId": 1 };\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "POST",\n url: "/posts",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpPutQuery', + type: 'http', + key: 'jsonplaceholder_renamed', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = { "id": 1, "title": "foo", "body": "bar", "userId": 1 };\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "PUT",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpDeleteQuery', + type: 'http', + key: 'jsonplaceholder_renamed', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = {};\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "DELETE",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + ], + sqlSnippets: [ + { + key: 'role_columns', + value: 'id, description', + }, + ], + }, + }, + }; + + const queryDashboardContentReponse = await server + .post('/dashboard_content/create') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(queryDashboardContentRequest); + + dashboardContentId = queryDashboardContentReponse.body.id; + }); + + afterAll(async () => { + if (!dashboardDataSource.isInitialized) { + await dashboardDataSource.initialize(); + } + await dashboardDataSource.getRepository(Dashboard).delete(dashboardId); + await dashboardDataSource.destroy(); }); describe('query', () => { @@ -26,54 +114,32 @@ describe('QueryController', () => { const query: QueryRequest = { type: 'postgresql', key: 'preset', - query: 'SELECT * FROM role ORDER BY id ASC', + query: "SELECT id, description FROM role WHERE id = 'SUPERADMIN' AND true", + content_id: dashboardContentId, + query_id: 'pgQuery', + params: { filters: { role_id: "'SUPERADMIN'" }, context: { true: 'true' } }, }; const response = await server.post('/query').set('Authorization', `Bearer ${superadminLogin.token}`).send(query); - expect(response.body).toMatchObject([ - { - id: FIXED_ROLE_TYPES.ADMIN, - description: - 'Can view and create dashboards. Can add and delete datasources. Can add users except other admins', - permissions: FIXED_ROLE_PERMISSIONS.ADMIN, - }, - { - id: FIXED_ROLE_TYPES.AUTHOR, - description: 'Can view and create dashboards', - permissions: FIXED_ROLE_PERMISSIONS.AUTHOR, - }, - { - id: FIXED_ROLE_TYPES.INACTIVE, - description: 'Disabled user. Can not login', - permissions: [], - }, - { - id: FIXED_ROLE_TYPES.READER, - description: 'Can view dashboards', - permissions: FIXED_ROLE_PERMISSIONS.READER, - }, - { - id: FIXED_ROLE_TYPES.SUPERADMIN, - description: 'Can do everything', - permissions: FIXED_ROLE_PERMISSIONS.SUPERADMIN, - }, - ]); + expect(response.body).toMatchObject([{ id: 'SUPERADMIN', description: 'Can do everything' }]); }); it('should query http successfully with GET', async () => { - const httpParams: HttpParams = { - host: '', - method: 'GET', - data: {}, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; const query: QueryRequest = { type: 'http', key: 'jsonplaceholder_renamed', - query: JSON.stringify(httpParams), + query: JSON.stringify({ + host: '', + method: 'GET', + data: {}, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + content_id: dashboardContentId, + query_id: 'httpGetQuery', + params: { filters: {}, context: {} }, }; const response = await server.post('/query').set('Authorization', `Bearer ${superadminLogin.token}`).send(query); @@ -91,18 +157,20 @@ describe('QueryController', () => { }); it('should query http successfully with POST', async () => { - const httpParams: HttpParams = { - host: '', - method: 'POST', - data: { title: 'foo', body: 'bar', userId: 1 }, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts', - }; const query: QueryRequest = { type: 'http', key: 'jsonplaceholder_renamed', - query: JSON.stringify(httpParams), + query: JSON.stringify({ + host: '', + method: 'POST', + data: { title: 'foo', body: 'bar', userId: 1 }, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts', + }), + content_id: dashboardContentId, + query_id: 'httpPostQuery', + params: { filters: {}, context: {} }, }; const response = await server.post('/query').set('Authorization', `Bearer ${superadminLogin.token}`).send(query); @@ -111,18 +179,20 @@ describe('QueryController', () => { }); it('should query http successfully with PUT', async () => { - const httpParams: HttpParams = { - host: '', - method: 'PUT', - data: { id: 1, title: 'foo', body: 'bar', userId: 1 }, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; const query: QueryRequest = { type: 'http', key: 'jsonplaceholder_renamed', - query: JSON.stringify(httpParams), + query: JSON.stringify({ + host: '', + method: 'PUT', + data: { id: 1, title: 'foo', body: 'bar', userId: 1 }, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + content_id: dashboardContentId, + query_id: 'httpPutQuery', + params: { filters: {}, context: {} }, }; const response = await server.post('/query').set('Authorization', `Bearer ${superadminLogin.token}`).send(query); @@ -131,18 +201,20 @@ describe('QueryController', () => { }); it('should query http successfully with DELETE', async () => { - const httpParams: HttpParams = { - host: '', - method: 'DELETE', - data: {}, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; const query: QueryRequest = { type: 'http', key: 'jsonplaceholder_renamed', - query: JSON.stringify(httpParams), + query: JSON.stringify({ + host: '', + method: 'DELETE', + data: {}, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + content_id: dashboardContentId, + query_id: 'httpDeleteQuery', + params: { filters: {}, context: {} }, }; const response = await server.post('/query').set('Authorization', `Bearer ${superadminLogin.token}`).send(query); @@ -150,4 +222,195 @@ describe('QueryController', () => { expect(response.body).toMatchObject({}); }); }); + + describe('queryStructure', () => { + it('query_type = TABLES', async () => { + const query: QueryStructureRequest = { + query_type: 'TABLES', + type: 'postgresql', + key: 'preset', + table_schema: '', + table_name: '', + }; + + const response = await server + .post('/query/structure') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(query); + + expect(response.body.length).toEqual(222); + expect(response.body[212]).toMatchObject({ + table_schema: 'public', + table_name: 'dashboard', + table_type: 'BASE TABLE', + }); + }); + + it('query_type = COLUMNS', async () => { + const query: QueryStructureRequest = { + query_type: 'COLUMNS', + type: 'postgresql', + key: 'preset', + table_schema: 'public', + table_name: 'dashboard', + }; + + const response = await server + .post('/query/structure') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(query); + + expect(response.body).toMatchObject([ + { + ordinal_position: 1, + column_key: 'P', + column_key_text: 'PRIMARY KEY (id)', + column_name: 'id', + column_type: 'uuid', + is_nullable: 'NO', + column_default: 'gen_random_uuid()', + column_comment: null, + }, + { + ordinal_position: 2, + column_key: null, + column_key_text: null, + column_name: 'name', + column_type: 'character varying', + is_nullable: 'NO', + column_default: null, + column_comment: null, + }, + { + ordinal_position: 4, + column_key: null, + column_key_text: null, + column_name: 'create_time', + column_type: 'timestamp with time zone', + is_nullable: 'NO', + column_default: 'CURRENT_TIMESTAMP', + column_comment: null, + }, + { + ordinal_position: 5, + column_key: null, + column_key_text: null, + column_name: 'update_time', + column_type: 'timestamp with time zone', + is_nullable: 'NO', + column_default: 'CURRENT_TIMESTAMP', + column_comment: null, + }, + { + ordinal_position: 6, + column_key: null, + column_key_text: null, + column_name: 'is_removed', + column_type: 'boolean', + is_nullable: 'NO', + column_default: 'false', + column_comment: null, + }, + { + ordinal_position: 7, + column_key: null, + column_key_text: null, + column_name: 'is_preset', + column_type: 'boolean', + is_nullable: 'NO', + column_default: 'false', + column_comment: null, + }, + { + ordinal_position: 8, + column_key: null, + column_key_text: null, + column_name: 'group', + column_type: 'character varying', + is_nullable: 'NO', + column_default: "''::character varying", + column_comment: null, + }, + { + ordinal_position: 9, + column_key: 'F', + column_key_text: 'FOREIGN KEY (content_id) REFERENCES dashboard_content(id) ON DELETE SET NULL', + column_name: 'content_id', + column_type: 'uuid', + is_nullable: 'YES', + column_default: null, + column_comment: null, + }, + ]); + }); + + it('query_type = DATA', async () => { + const query: QueryStructureRequest = { + query_type: 'DATA', + type: 'postgresql', + key: 'preset', + table_schema: 'public', + table_name: 'dashboard', + }; + + const response = await server + .post('/query/structure') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(query); + + expect(response.body.length).toEqual(4); + }); + + it('query_type = COUNT', async () => { + const query: QueryStructureRequest = { + query_type: 'COUNT', + type: 'postgresql', + key: 'preset', + table_schema: 'public', + table_name: 'dashboard', + }; + + const response = await server + .post('/query/structure') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(query); + + expect(response.body).toMatchObject([{ total: '4' }]); + }); + + it('query_type = INDEXES', async () => { + const query: QueryStructureRequest = { + query_type: 'INDEXES', + type: 'postgresql', + key: 'preset', + table_schema: 'public', + table_name: 'dashboard', + }; + + const response = await server + .post('/query/structure') + .set('Authorization', `Bearer ${superadminLogin.token}`) + .send(query); + + expect(response.body).toMatchObject([ + { + index_name: 'dashboard_pkey', + index_algorithm: 'BTREE', + is_unique: true, + index_definition: 'CREATE UNIQUE INDEX dashboard_pkey ON public.dashboard USING btree (id)', + condition: '', + comment: null, + }, + { + index_name: 'dashboard_name_preset_idx', + index_algorithm: 'BTREE', + is_unique: true, + index_definition: + 'CREATE UNIQUE INDEX dashboard_name_preset_idx ON public.dashboard USING btree (name, is_preset) WHERE (is_removed = false)', + condition: '(is_removed = false)', + comment: null, + }, + ]); + }); + }); }); diff --git a/api/tests/e2e/11_dashboard_content.test.ts b/api/tests/e2e/11_dashboard_content.test.ts index 38d9795d1..e80d997df 100644 --- a/api/tests/e2e/11_dashboard_content.test.ts +++ b/api/tests/e2e/11_dashboard_content.test.ts @@ -58,13 +58,13 @@ describe('DashboardContentController', () => { const presetContent1Data = new DashboardContent(); presetContent1Data.dashboard_id = presetDashboard.id; presetContent1Data.name = 'presetContent1'; - presetContent1Data.content = {}; + presetContent1Data.content = { version: '', definition: { queries: [], sqlSnippets: [] } }; presetDashboardContent1 = await dashboardDataSource.getRepository(DashboardContent).save(presetContent1Data); const presetContent2Data = new DashboardContent(); presetContent2Data.dashboard_id = presetDashboard.id; presetContent2Data.name = 'presetContent2'; - presetContent2Data.content = {}; + presetContent2Data.content = { version: '', definition: { queries: [], sqlSnippets: [] } }; presetDashboardContent2 = await dashboardDataSource.getRepository(DashboardContent).save(presetContent2Data); const dashboard1Data = new Dashboard(); @@ -97,7 +97,7 @@ describe('DashboardContentController', () => { const request1: DashboardContentCreateRequest = { dashboard_id: dashboard1.id, name: 'dashboard1_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response1 = await server @@ -111,13 +111,13 @@ describe('DashboardContentController', () => { id: response1.body.id, dashboard_id: dashboard1.id, name: 'dashboard1_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }); const request2: DashboardContentCreateRequest = { dashboard_id: dashboard1.id, name: 'dashboard1_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response2 = await server @@ -131,13 +131,13 @@ describe('DashboardContentController', () => { id: response2.body.id, dashboard_id: dashboard1.id, name: 'dashboard1_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }); const request3: DashboardContentCreateRequest = { dashboard_id: dashboard2.id, name: 'dashboard2_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response3 = await server @@ -151,13 +151,13 @@ describe('DashboardContentController', () => { id: response3.body.id, dashboard_id: dashboard2.id, name: 'dashboard2_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }); const request4: DashboardContentCreateRequest = { dashboard_id: dashboard2.id, name: 'dashboard2_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response4 = await server @@ -171,7 +171,7 @@ describe('DashboardContentController', () => { id: response4.body.id, dashboard_id: dashboard2.id, name: 'dashboard2_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }); }); @@ -179,7 +179,7 @@ describe('DashboardContentController', () => { const req: DashboardContentCreateRequest = { dashboard_id: dashboard1.id, name: 'dashboard1_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response = await server @@ -218,13 +218,13 @@ describe('DashboardContentController', () => { id: response1.body.data[0].id, dashboard_id: dashboard1.id, name: 'dashboard1_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, { id: response1.body.data[1].id, dashboard_id: dashboard1.id, name: 'dashboard1_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); @@ -249,13 +249,13 @@ describe('DashboardContentController', () => { id: response2.body.data[0].id, dashboard_id: dashboard2.id, name: 'dashboard2_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, { id: response2.body.data[1].id, dashboard_id: dashboard2.id, name: 'dashboard2_content2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); @@ -283,7 +283,7 @@ describe('DashboardContentController', () => { id: response.body.data[0].id, dashboard_id: dashboard1.id, name: 'dashboard1_content1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); @@ -326,7 +326,7 @@ describe('DashboardContentController', () => { const query1: DashboardContentUpdateRequest = { id: dashboard1Content1.id, name: 'dashboard1_content1_updated', - content: { tmp: 'tmp' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response1 = await server @@ -343,7 +343,7 @@ describe('DashboardContentController', () => { const query2: DashboardContentUpdateRequest = { id: dashboard2Content1.id, name: 'dashboard2_content1_updated', - content: { tmp: 'tmp' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response2 = await server @@ -360,7 +360,7 @@ describe('DashboardContentController', () => { const query3: DashboardContentUpdateRequest = { id: dashboard2Content2.id, name: 'dashboard2_content2_updated', - content: { tmp: 'tmp' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response3 = await server @@ -377,7 +377,7 @@ describe('DashboardContentController', () => { const query4: DashboardContentUpdateRequest = { id: dashboard1Content2.id, name: 'dashboard1_content2_updated', - content: { tmp: 'tmp1' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response4 = await server @@ -412,7 +412,7 @@ describe('DashboardContentController', () => { const query: DashboardContentUpdateRequest = { id: presetDashboardContent1.id, name: 'presetContent1_updated', - content: { tmp: 'tmp' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const response = await server @@ -424,7 +424,7 @@ describe('DashboardContentController', () => { expect(response.body).toMatchObject({ ...omitFields(presetDashboardContent1, ['create_time', 'update_time']), name: 'presetContent1_updated', - content: { tmp: 'tmp' }, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }); }); @@ -552,13 +552,13 @@ describe('DashboardContentController', () => { id: response1.body.data[0].id, dashboard_id: dashboard2.id, name: 'dashboard2_content1_updated', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, { id: response1.body.data[1].id, dashboard_id: dashboard2.id, name: 'dashboard2_content2_updated', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); diff --git a/api/tests/e2e/12_dashboard_content_changelog.test.ts b/api/tests/e2e/12_dashboard_content_changelog.test.ts index 0964ad115..8541f5cb2 100644 --- a/api/tests/e2e/12_dashboard_content_changelog.test.ts +++ b/api/tests/e2e/12_dashboard_content_changelog.test.ts @@ -49,15 +49,9 @@ describe('DashboardChangelogController', () => { expect(response.body.data[0].diff).toContain('diff --git a/data.json b/data.json'); expect(response.body.data[0].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(response.body.data[0].diff).toContain('@@ -1,6 +1,8 @@'); + expect(response.body.data[0].diff).toContain('@@ -1,7 +1,7 @@'); expect(response.body.data[0].diff).toContain( - '-\t"name": "dashboard1_content2",\n' + - '-\t"content": {}\n' + - '+\t"name": "dashboard1_content2_updated",\n' + - '+\t"content": {\n' + - '+\t\t"tmp": "tmp1"\n' + - '+\t}\n' + - ' }\n', + '-\t"name": "dashboard1_content2",\n' + '+\t"name": "dashboard1_content2_updated",\n', ); changelogDashboardContentId = response.body.data[0].dashboard_content_id; @@ -90,15 +84,9 @@ describe('DashboardChangelogController', () => { expect(response.body.data[0].diff).toContain('diff --git a/data.json b/data.json'); expect(response.body.data[0].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(response.body.data[0].diff).toContain('@@ -1,6 +1,8 @@'); + expect(response.body.data[0].diff).toContain('@@ -1,7 +1,7 @@'); expect(response.body.data[0].diff).toContain( - '-\t"name": "dashboard1_content2",\n' + - '-\t"content": {}\n' + - '+\t"name": "dashboard1_content2_updated",\n' + - '+\t"content": {\n' + - '+\t\t"tmp": "tmp1"\n' + - '+\t}\n' + - ' }\n', + '-\t"name": "dashboard1_content2",\n' + '+\t"name": "dashboard1_content2_updated",\n', ); }); }); diff --git a/api/tests/e2e/jest.setup.ts b/api/tests/e2e/jest.setup.ts index 7ed9dbe7d..d89bdbdc7 100644 --- a/api/tests/e2e/jest.setup.ts +++ b/api/tests/e2e/jest.setup.ts @@ -8,6 +8,7 @@ module.exports = async (globalConfig) => { process.env.WEBSITE_LOGO_URL_EN = 'WEBSITE_LOGO_URL_EN'; process.env.WEBSITE_LOGO_JUMP_URL = '/WEBSITE_LOGO_JUMP_URL'; process.env.WEBSITE_FAVICON_URL = '/WEBSITE_FAVICON_URL'; + process.env.ENABLE_QUERY_PARSER = '1'; dashboardDataSource.setOptions({ url: process.env.END_2_END_TEST_PG_URL! }); diff --git a/api/tests/integration/06_query.test.ts b/api/tests/integration/06_query.test.ts index 480d01dd8..e4806fac3 100644 --- a/api/tests/integration/06_query.test.ts +++ b/api/tests/integration/06_query.test.ts @@ -1,8 +1,6 @@ import { connectionHook } from './jest.util'; import { QueryService } from '~/services/query.service'; -import { HttpParams } from '~/api_models/query'; -import * as validation from '~/middleware/validation'; -import { FIXED_ROLE_PERMISSIONS, FIXED_ROLE_TYPES } from '~/services/role.service'; +import { DEFAULT_LANGUAGE } from '~/utils/constants'; describe('QueryService', () => { connectionHook(); @@ -14,49 +12,39 @@ describe('QueryService', () => { describe('query', () => { it('should query pg successfully', async () => { - const results = await queryService.query('postgresql', 'pg', 'SELECT * FROM role ORDER BY id ASC', {}); - expect(results).toMatchObject([ - { - id: FIXED_ROLE_TYPES.ADMIN, - description: - 'Can view and create dashboards. Can add and delete datasources. Can add users except other admins', - permissions: FIXED_ROLE_PERMISSIONS.ADMIN, - }, - { - id: FIXED_ROLE_TYPES.AUTHOR, - description: 'Can view and create dashboards', - permissions: FIXED_ROLE_PERMISSIONS.AUTHOR, - }, - { - id: FIXED_ROLE_TYPES.INACTIVE, - description: 'Disabled user. Can not login', - permissions: [], - }, - { - id: FIXED_ROLE_TYPES.READER, - description: 'Can view dashboards', - permissions: FIXED_ROLE_PERMISSIONS.READER, - }, - { - id: FIXED_ROLE_TYPES.SUPERADMIN, - description: 'Can do everything', - permissions: FIXED_ROLE_PERMISSIONS.SUPERADMIN, - }, - ]); + const results = await queryService.query( + 'postgresql', + 'pg', + "SELECT id, description FROM role WHERE id = 'SUPERADMIN' AND true", + '9afa4842-77ef-4b19-8a53-034cb41ee7f6', + 'pgQuery1', + { filters: { role_id: "'SUPERADMIN'" }, context: { true: 'true' } }, + {}, + true, + DEFAULT_LANGUAGE, + ); + expect(results).toMatchObject([{ id: 'SUPERADMIN', description: 'Can do everything' }]); }); it('should query http successfully with GET', async () => { - const query: HttpParams = { - host: '', - method: 'GET', - data: {}, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; - const validateClass = jest.spyOn(validation, 'validateClass'); - validateClass.mockReturnValueOnce(query); - const results = await queryService.query('http', 'jsonplaceholder', JSON.stringify(query), {}); + const results = await queryService.query( + 'http', + 'jsonplaceholder', + JSON.stringify({ + host: '', + method: 'GET', + data: {}, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + '9afa4842-77ef-4b19-8a53-034cb41ee7f6', + 'httpGetQuery', + { filters: {}, context: {} }, + {}, + true, + DEFAULT_LANGUAGE, + ); expect(results).toMatchObject({ userId: 1, id: 1, @@ -70,48 +58,200 @@ describe('QueryService', () => { }); it('should query http successfully with POST', async () => { - const query: HttpParams = { - host: '', - method: 'POST', - data: { title: 'foo', body: 'bar', userId: 1 }, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts', - }; - const validateClass = jest.spyOn(validation, 'validateClass'); - validateClass.mockReturnValueOnce(query); - const results = await queryService.query('http', 'jsonplaceholder', JSON.stringify(query), {}); + const results = await queryService.query( + 'http', + 'jsonplaceholder', + JSON.stringify({ + host: '', + method: 'POST', + data: { title: 'foo', body: 'bar', userId: 1 }, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts', + }), + '9afa4842-77ef-4b19-8a53-034cb41ee7f6', + 'httpPostQuery', + { filters: {}, context: {} }, + {}, + true, + DEFAULT_LANGUAGE, + ); expect(results).toMatchObject({ title: 'foo', body: 'bar', userId: 1, id: 101 }); }); it('should query http successfully with PUT', async () => { - const query: HttpParams = { - host: '', - method: 'PUT', - data: { id: 1, title: 'foo', body: 'bar', userId: 1 }, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; - const validateClass = jest.spyOn(validation, 'validateClass'); - validateClass.mockReturnValueOnce(query); - const results = await queryService.query('http', 'jsonplaceholder', JSON.stringify(query), {}); + const results = await queryService.query( + 'http', + 'jsonplaceholder', + JSON.stringify({ + host: '', + method: 'PUT', + data: { id: 1, title: 'foo', body: 'bar', userId: 1 }, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + '9afa4842-77ef-4b19-8a53-034cb41ee7f6', + 'httpPutQuery', + { filters: {}, context: {} }, + {}, + true, + DEFAULT_LANGUAGE, + ); expect(results).toMatchObject({ title: 'foo', body: 'bar', userId: 1, id: 1 }); }); it('should query http successfully with DELETE', async () => { - const query: HttpParams = { - host: '', - method: 'DELETE', - data: {}, - params: {}, - headers: { 'Content-Type': 'application/json' }, - url: '/posts/1', - }; - const validateClass = jest.spyOn(validation, 'validateClass'); - validateClass.mockReturnValueOnce(query); - const results = await queryService.query('http', 'jsonplaceholder', JSON.stringify(query), {}); + const results = await queryService.query( + 'http', + 'jsonplaceholder', + JSON.stringify({ + host: '', + method: 'DELETE', + data: {}, + params: {}, + headers: { 'Content-Type': 'application/json' }, + url: '/posts/1', + }), + '9afa4842-77ef-4b19-8a53-034cb41ee7f6', + 'httpDeleteQuery', + { filters: {}, context: {} }, + {}, + true, + DEFAULT_LANGUAGE, + ); expect(results).toMatchObject({}); }); }); + + describe('queryStructure', () => { + it('TABLES', async () => { + const results = await queryService.queryStructure('TABLES', 'postgresql', 'pg', '', ''); + expect(results.length).toEqual(222); + expect(results[212]).toMatchObject({ + table_schema: 'public', + table_name: 'dashboard', + table_type: 'BASE TABLE', + }); + }); + + it('COLUMNS', async () => { + const results = await queryService.queryStructure('COLUMNS', 'postgresql', 'pg', 'public', 'dashboard'); + expect(results).toMatchObject([ + { + ordinal_position: 1, + column_key: 'P', + column_key_text: 'PRIMARY KEY (id)', + column_name: 'id', + column_type: 'uuid', + is_nullable: 'NO', + column_default: 'gen_random_uuid()', + column_comment: null, + }, + { + ordinal_position: 2, + column_key: null, + column_key_text: null, + column_name: 'name', + column_type: 'character varying', + is_nullable: 'NO', + column_default: null, + column_comment: null, + }, + { + ordinal_position: 4, + column_key: null, + column_key_text: null, + column_name: 'create_time', + column_type: 'timestamp with time zone', + is_nullable: 'NO', + column_default: 'CURRENT_TIMESTAMP', + column_comment: null, + }, + { + ordinal_position: 5, + column_key: null, + column_key_text: null, + column_name: 'update_time', + column_type: 'timestamp with time zone', + is_nullable: 'NO', + column_default: 'CURRENT_TIMESTAMP', + column_comment: null, + }, + { + ordinal_position: 6, + column_key: null, + column_key_text: null, + column_name: 'is_removed', + column_type: 'boolean', + is_nullable: 'NO', + column_default: 'false', + column_comment: null, + }, + { + ordinal_position: 7, + column_key: null, + column_key_text: null, + column_name: 'is_preset', + column_type: 'boolean', + is_nullable: 'NO', + column_default: 'false', + column_comment: null, + }, + { + ordinal_position: 8, + column_key: null, + column_key_text: null, + column_name: 'group', + column_type: 'character varying', + is_nullable: 'NO', + column_default: "''::character varying", + column_comment: null, + }, + { + ordinal_position: 9, + column_key: 'F', + column_key_text: 'FOREIGN KEY (content_id) REFERENCES dashboard_content(id) ON DELETE SET NULL', + column_name: 'content_id', + column_type: 'uuid', + is_nullable: 'YES', + column_default: null, + column_comment: null, + }, + ]); + }); + + it('DATA', async () => { + const results = await queryService.queryStructure('DATA', 'postgresql', 'pg', 'public', 'dashboard'); + expect(results.length).toEqual(3); + }); + + it('COUNT', async () => { + const results = await queryService.queryStructure('COUNT', 'postgresql', 'pg', 'public', 'dashboard'); + expect(results).toMatchObject([{ total: '3' }]); + }); + + it('INDEXES', async () => { + const results = await queryService.queryStructure('INDEXES', 'postgresql', 'pg', 'public', 'dashboard'); + expect(results).toMatchObject([ + { + index_name: 'dashboard_pkey', + index_algorithm: 'BTREE', + is_unique: true, + index_definition: 'CREATE UNIQUE INDEX dashboard_pkey ON public.dashboard USING btree (id)', + condition: '', + comment: null, + }, + { + index_name: 'dashboard_name_preset_idx', + index_algorithm: 'BTREE', + is_unique: true, + index_definition: + 'CREATE UNIQUE INDEX dashboard_name_preset_idx ON public.dashboard USING btree (name, is_preset) WHERE (is_removed = false)', + condition: '(is_removed = false)', + comment: null, + }, + ]); + }); + }); }); diff --git a/api/tests/integration/07_job.test.ts b/api/tests/integration/07_job.test.ts index 96eb84587..103192ba0 100644 --- a/api/tests/integration/07_job.test.ts +++ b/api/tests/integration/07_job.test.ts @@ -260,7 +260,7 @@ describe('JobService', () => { result: { affected_dashboard_contents: [ { - queries: ['httpQuery1'], + queries: ['httpGetQuery', 'httpPostQuery', 'httpPutQuery', 'httpDeleteQuery'], contentId: results.data[5].result['affected_dashboard_contents'][0].contentId, }, { @@ -476,7 +476,7 @@ describe('JobService', () => { result: { affected_dashboard_contents: [ { - queries: ['httpQuery1'], + queries: ['httpGetQuery', 'httpPostQuery', 'httpPutQuery', 'httpDeleteQuery'], contentId: results.data[5].result['affected_dashboard_contents'][0].contentId, }, { diff --git a/api/tests/integration/11_dashboard_content.test.ts b/api/tests/integration/11_dashboard_content.test.ts index 0d16b2148..db8e5f159 100644 --- a/api/tests/integration/11_dashboard_content.test.ts +++ b/api/tests/integration/11_dashboard_content.test.ts @@ -68,25 +68,30 @@ describe('DashboardContentService', () => { dashboardContent1 = await dashboardContentService.create( tempDashboard.id, 'dashboardContent1', - {}, + { version: '', definition: { queries: [], sqlSnippets: [] } }, DEFAULT_LANGUAGE, ); dashboardContent2 = await dashboardContentService.create( tempDashboard.id, 'dashboardContent2', - {}, + { version: '', definition: { queries: [], sqlSnippets: [] } }, DEFAULT_LANGUAGE, ); dashboardContent3 = await dashboardContentService.create( tempDashboard.id, 'dashboardContent3', - {}, + { version: '', definition: { queries: [], sqlSnippets: [] } }, DEFAULT_LANGUAGE, ); }); it('should fail if duplicate name', async () => { await expect( - dashboardContentService.create(tempDashboard.id, 'dashboardContent1', {}, DEFAULT_LANGUAGE), + dashboardContentService.create( + tempDashboard.id, + 'dashboardContent1', + { version: '', definition: { queries: [], sqlSnippets: [] } }, + DEFAULT_LANGUAGE, + ), ).rejects.toThrowError( new ApiError(BAD_REQUEST, { message: 'A dashboard content with that name already exists' }), ); @@ -112,19 +117,19 @@ describe('DashboardContentService', () => { id: dashboardContent1.id, dashboard_id: tempDashboard.id, name: 'dashboardContent1', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, { id: dashboardContent2.id, dashboard_id: tempDashboard.id, name: 'dashboardContent2', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, { id: dashboardContent3.id, dashboard_id: tempDashboard.id, name: 'dashboardContent3', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); @@ -145,7 +150,7 @@ describe('DashboardContentService', () => { id: dashboardContent3.id, dashboard_id: tempDashboard.id, name: 'dashboardContent3', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }, ], }); diff --git a/api/tests/integration/12_dashboard_content_changelog.test.ts b/api/tests/integration/12_dashboard_content_changelog.test.ts index 644c5c8df..32280a5f4 100644 --- a/api/tests/integration/12_dashboard_content_changelog.test.ts +++ b/api/tests/integration/12_dashboard_content_changelog.test.ts @@ -21,9 +21,9 @@ describe('DashboardContentChangelogService', () => { expect(results.data[0].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[0].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[0].diff).toContain('@@ -7,7 +7,7 @@'); + expect(results.data[0].diff).toContain('@@ -8,7 +8,7 @@'); expect(results.data[0].diff).toContain('-\t\t\t\t\t"key": "pg",\n' + '+\t\t\t\t\t"key": "pg_renamed",\n'); - expect(results.data[0].diff).toContain('@@ -17,5 +17,9 @@'); + expect(results.data[0].diff).toContain('@@ -49,5 +49,9 @@'); expect(results.data[0].diff).toContain( '-\t}\n' + '+\t},\n' + @@ -36,9 +36,9 @@ describe('DashboardContentChangelogService', () => { expect(results.data[1].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[1].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[1].diff).toContain('@@ -7,7 +7,7 @@'); + expect(results.data[1].diff).toContain('@@ -8,7 +8,7 @@'); expect(results.data[1].diff).toContain('-\t\t\t\t\t"key": "pg",\n' + '+\t\t\t\t\t"key": "pg_renamed",\n'); - expect(results.data[1].diff).toContain('@@ -17,5 +17,9 @@'); + expect(results.data[1].diff).toContain('@@ -23,5 +23,9 @@'); expect(results.data[1].diff).toContain( '-\t}\n' + '+\t},\n' + @@ -51,14 +51,14 @@ describe('DashboardContentChangelogService', () => { expect(results.data[2].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[2].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[2].diff).toContain('@@ -12,7 +12,7 @@'); + expect(results.data[2].diff).toContain('@@ -15,28 +15,28 @@'); expect(results.data[2].diff).toContain( '-\t\t\t\t\t"key": "jsonplaceholder",\n' + '+\t\t\t\t\t"key": "jsonplaceholder_renamed",\n', ); expect(results.data[3].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[3].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[3].diff).toContain('@@ -12,7 +12,7 @@'); + expect(results.data[3].diff).toContain('@@ -15,7 +15,7 @@'); expect(results.data[3].diff).toContain( '-\t\t\t\t\t"key": "jsonplaceholder",\n' + '+\t\t\t\t\t"key": "jsonplaceholder_renamed",\n', ); @@ -78,9 +78,9 @@ describe('DashboardContentChangelogService', () => { expect(results.data[0].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[0].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[0].diff).toContain('@@ -7,7 +7,7 @@'); + expect(results.data[0].diff).toContain('@@ -8,7 +8,7 @@'); expect(results.data[0].diff).toContain('-\t\t\t\t\t"key": "pg",\n' + '+\t\t\t\t\t"key": "pg_renamed",\n'); - expect(results.data[0].diff).toContain('@@ -17,5 +17,9 @@'); + expect(results.data[0].diff).toContain('@@ -49,5 +49,9 @@'); expect(results.data[0].diff).toContain( '-\t}\n' + '+\t},\n' + @@ -93,7 +93,7 @@ describe('DashboardContentChangelogService', () => { expect(results.data[1].diff).toContain('diff --git a/data.json b/data.json'); expect(results.data[1].diff).toContain('--- a/data.json\n' + '+++ b/data.json'); - expect(results.data[1].diff).toContain('@@ -12,7 +12,7 @@'); + expect(results.data[1].diff).toContain('@@ -15,28 +15,28 @@'); expect(results.data[1].diff).toContain( '-\t\t\t\t\t"key": "jsonplaceholder",\n' + '+\t\t\t\t\t"key": "jsonplaceholder_renamed",\n', ); diff --git a/api/tests/integration/constants.ts b/api/tests/integration/constants.ts index 567f08f09..9038f995d 100644 --- a/api/tests/integration/constants.ts +++ b/api/tests/integration/constants.ts @@ -172,17 +172,53 @@ export const dashboardContents: DashboardContent[] = [ dashboard_id: '63c52cf7-0783-40fb-803a-68abc6564de0', name: 'dashboard1', content: { + version: '', definition: { queries: [ { id: 'pgQuery1', type: 'postgresql', key: 'pg', + sql: 'SELECT ${sql_snippets.role_columns} FROM role WHERE id = ${filters.role_id} AND ${context.true}', + pre_process: '', }, { - id: 'httpQuery1', + id: 'httpGetQuery', type: 'http', key: 'jsonplaceholder', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = {};\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "GET",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpPostQuery', + type: 'http', + key: 'jsonplaceholder', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = { "title": "foo", "body": "bar", "userId": 1 };\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "POST",\n url: "/posts",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpPutQuery', + type: 'http', + key: 'jsonplaceholder', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = { "id": 1, "title": "foo", "body": "bar", "userId": 1 };\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "PUT",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + { + id: 'httpDeleteQuery', + type: 'http', + key: 'jsonplaceholder', + sql: '', + pre_process: + 'function build_request({ context, filters }, utils) {\n const data = {};\n const headers = { "Content-Type": "application/json" };\n\n return {\n method: "DELETE",\n url: "/posts/1",\n params: {},\n headers,\n data,\n };\n}\n', + }, + ], + sqlSnippets: [ + { + key: 'role_columns', + value: 'id, description', }, ], }, @@ -195,19 +231,25 @@ export const dashboardContents: DashboardContent[] = [ dashboard_id: '173b84d2-7ed9-4d1a-a386-e68a6cce192b', name: 'dashboard2', content: { + version: '', definition: { queries: [ { id: 'pgQuery2', type: 'postgresql', key: 'pg', + sql: '', + pre_process: '', }, { id: 'httpQuery2', type: 'http', key: 'jsonplaceholder', + sql: '', + pre_process: '', }, ], + sqlSnippets: [], }, }, create_time: new Date(), diff --git a/api/tests/integration/jest.setup.ts b/api/tests/integration/jest.setup.ts index 7b85d66eb..c4ca8b4f2 100644 --- a/api/tests/integration/jest.setup.ts +++ b/api/tests/integration/jest.setup.ts @@ -8,6 +8,7 @@ module.exports = async (globalConfig) => { process.env.WEBSITE_LOGO_URL_EN = 'WEBSITE_LOGO_URL_EN'; process.env.WEBSITE_LOGO_JUMP_URL = '/WEBSITE_LOGO_JUMP_URL'; process.env.WEBSITE_FAVICON_URL = '/WEBSITE_FAVICON_URL'; + process.env.ENABLE_QUERY_PARSER = '1'; dashboardDataSource.setOptions({ url: process.env.INTEGRATION_TEST_PG_URL! }); diff --git a/api/tests/unit/validation.test.ts b/api/tests/unit/validation.test.ts index 5b74ade72..7bcdac3f6 100644 --- a/api/tests/unit/validation.test.ts +++ b/api/tests/unit/validation.test.ts @@ -23,7 +23,7 @@ import { DataSourceUpdateRequest, } from '~/api_models/datasource'; import { JobListRequest, JobRunRequest } from '~/api_models/job'; -import { QueryRequest } from '~/api_models/query'; +import { QueryRequest, QueryStructureRequest } from '~/api_models/query'; import { ConfigGetRequest, ConfigUpdateRequest } from '~/api_models/config'; import { DashboardChangelogListRequest } from '~/api_models/dashboard_changelog'; import { @@ -931,7 +931,7 @@ describe('DashboardContentCreateRequest', () => { const data: DashboardContentCreateRequest = { dashboard_id: crypto.randomUUID(), name: 'test', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const result = validateClass(DashboardContentCreateRequest, data); expect(result).toMatchObject(data); @@ -1010,7 +1010,7 @@ describe('DashboardContentUpdateRequest', () => { const data: DashboardContentUpdateRequest = { id: crypto.randomUUID(), name: 'test', - content: {}, + content: { version: '', definition: { queries: [], sqlSnippets: [] } }, }; const result = validateClass(DashboardContentUpdateRequest, data); expect(result).toMatchObject(data); @@ -1764,9 +1764,15 @@ describe('JobRunRequest', () => { describe('QueryRequest', () => { it('Should have no validation errors', () => { const data: QueryRequest = { - type: 'http', - key: 'test', + type: 'postgresql', + key: '', query: '', + content_id: crypto.randomUUID(), + query_id: 'test', + params: { + context: {}, + filters: {}, + }, }; const result = validateClass(QueryRequest, data); @@ -1787,9 +1793,7 @@ describe('QueryRequest', () => { value: undefined, property: 'type', children: [], - constraints: { - isIn: 'type must be one of the following values: postgresql, mysql, http', - }, + constraints: { isIn: 'type must be one of the following values: postgresql, mysql, http' }, }, { target: {}, @@ -1805,6 +1809,139 @@ describe('QueryRequest', () => { children: [], constraints: { isString: 'query must be a string' }, }, + { + target: {}, + value: undefined, + property: 'content_id', + children: [], + constraints: { isString: 'content_id must be a string' }, + }, + { + target: {}, + value: undefined, + property: 'query_id', + children: [], + constraints: { isString: 'query_id must be a string' }, + }, + { + target: {}, + value: undefined, + property: 'params', + children: [], + constraints: { isObject: 'params must be an object' }, + }, + ]); + } + }); + + it('Should have validation errors if params variable is incorrect', () => { + const data = { + type: 'postgresql', + key: '', + query: '', + content_id: crypto.randomUUID(), + query_id: 'test', + params: {}, + }; + expect(() => validateClass(QueryRequest, data)).toThrow( + new ApiError(VALIDATION_FAILED, { message: `request body is incorrect` }), + ); + try { + validateClass(QueryRequest, data); + } catch (error) { + expect(error.detail.errors).toMatchObject([ + { + target: { + content_id: data.content_id, + query_id: 'test', + params: {}, + }, + value: {}, + property: 'params', + children: [ + { + target: {}, + value: undefined, + property: 'filters', + children: [], + constraints: { isObject: 'filters must be an object' }, + }, + { + target: {}, + value: undefined, + property: 'context', + children: [], + constraints: { isObject: 'context must be an object' }, + }, + ], + }, + ]); + } + }); +}); + +describe('QueryStructureRequest', () => { + it('Should have no validation errors', () => { + const data: QueryStructureRequest = { + query_type: 'TABLES', + type: 'postgresql', + key: '', + table_schema: '', + table_name: '', + limit: 20, + offset: 0, + }; + + const result = validateClass(QueryStructureRequest, data); + expect(result).toMatchObject(data); + }); + + it('Should have validation errors', () => { + const data = {}; + expect(() => validateClass(QueryStructureRequest, data)).toThrow( + new ApiError(VALIDATION_FAILED, { message: `request body is incorrect` }), + ); + try { + validateClass(QueryStructureRequest, data); + } catch (error) { + expect(error.detail.errors).toMatchObject([ + { + target: {}, + value: undefined, + property: 'query_type', + children: [], + constraints: { + isIn: 'query_type must be one of the following values: TABLES, COLUMNS, DATA, INDEXES, COUNT', + }, + }, + { + target: {}, + value: undefined, + property: 'type', + children: [], + constraints: { isIn: 'type must be one of the following values: postgresql, mysql' }, + }, + { + target: {}, + value: undefined, + property: 'key', + children: [], + constraints: { isString: 'key must be a string' }, + }, + { + target: {}, + value: undefined, + property: 'table_schema', + children: [], + constraints: { isString: 'table_schema must be a string' }, + }, + { + target: {}, + value: undefined, + property: 'table_name', + children: [], + constraints: { isString: 'table_name must be a string' }, + }, ]); } });