From d278a20f765089f993ad22843ae53aa90308f973 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Tue, 30 Apr 2024 13:11:46 -0600 Subject: [PATCH 1/2] Corrected content type for bulk operations Signed-off-by: Theo Truong --- CLIENT_GENERATOR_GUIDE.md | 4 +++- spec/namespaces/_core.yaml | 9 +++----- spec/namespaces/search_pipeline.yaml | 2 +- spec/namespaces/security.yaml | 26 ++++++++++++------------ tools/helpers.ts | 17 +++++++++++++++- tools/linter/components/NamespaceFile.ts | 4 ++-- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/CLIENT_GENERATOR_GUIDE.md b/CLIENT_GENERATOR_GUIDE.md index ceed01036..29077c229 100644 --- a/CLIENT_GENERATOR_GUIDE.md +++ b/CLIENT_GENERATOR_GUIDE.md @@ -45,7 +45,9 @@ def search(self, index=None, body=None): You will also encounter `x-overloaded-param: metric` for the `node_id` path parameter of the `GET /_nodes/{node_id}` operation in `nodes.info` action. This is a special case where the path parameter is overloaded to accept either a node ID or a metric name. When the user evokes the `client.nodes.info` method with either `metric` or `node_id` (but not both), the method will use the `GET /_nodes/{node_id}` operation. When evoked with both `metric` and `node_id`, it will use the `GET /_nodes/{node_id}/{metric}` operation. ## Handling Bulk Operations -Some operations accept a bulk of data in the request body. For example, the `bulk` action accepts a bulk of index, update, and delete operations on multiple documents. Unlike other operations where the request body is a **JSON object**, the request body for bulk operations is an **NDJSON** (i.e a [Newline-delimited JSON](https://github.com/ndjson/ndjson-spec)). When encountering this type of operation, the client must serialize the request body accordingly, and set the `Content-Type` header to `application/x-ndjson`. +Some operations accept a bulk of data in the request body. For example, the `bulk` action accepts a bulk of index, update, and delete operations on multiple documents. Unlike other operations where the request body is a **JSON object**, the request body for bulk operations is an **NDJSON** (i.e a [Newline-delimited JSON](https://github.com/ndjson/ndjson-spec)). + +In the spec, these request bodies have the `content` of `application/x-ndjson` and their schemas are of type array. In this situation, the client must serialize the request body accordingly, and set the `Content-Type` header to `application/x-ndjson`. ## Parameter Validation As of right now, most clients only validate whether required parameters are present. The clients do not validate the values of parameters against the enum values or regex patterns. This is to reduce performance overhead for the clients as the validation is already done on the server. However, the list of enum values and regex patterns are often written into the parameter description. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index add453f67..baa2f43e0 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2110,7 +2110,7 @@ components: requestBodies: bulk: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -2119,7 +2119,6 @@ components: - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction' - type: object description: The operation definition and data (action-data pairs), separated by newlines - x-serialize: bulk required: true clear_scroll: content: @@ -2220,23 +2219,21 @@ components: required: true msearch: content: - application/json: + application/x-ndjson: schema: type: array items: $ref: '../schemas/_core.msearch.yaml#/components/schemas/RequestItem' description: The request definitions (metadata-search request definition pairs), separated by newlines - x-serialize: bulk required: true msearch_template: content: - application/json: + application/x-ndjson: schema: type: array items: $ref: '../schemas/_core.msearch_template.yaml#/components/schemas/RequestItem' description: The request definitions (metadata-search request definition pairs), separated by newlines - x-serialize: bulk required: true mtermvectors: content: diff --git a/spec/namespaces/search_pipeline.yaml b/spec/namespaces/search_pipeline.yaml index 9eb60efeb..775e89b4b 100644 --- a/spec/namespaces/search_pipeline.yaml +++ b/spec/namespaces/search_pipeline.yaml @@ -141,4 +141,4 @@ components: in: query description: Operation timeout. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' \ No newline at end of file + $ref: '../schemas/_common.yaml#/components/schemas/Duration' diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 06a4d6bab..48b48993f 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -637,7 +637,7 @@ components: required: true security.patch_action_group: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -645,7 +645,7 @@ components: required: true security.patch_action_groups: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -653,7 +653,7 @@ components: required: true security.patch_audit_configuration: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -661,7 +661,7 @@ components: required: true security.patch_configuration: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -669,7 +669,7 @@ components: required: true security.patch_distinguished_names: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -677,7 +677,7 @@ components: required: true security.patch_role: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -685,7 +685,7 @@ components: required: true security.patch_role_mapping: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -693,7 +693,7 @@ components: required: true security.patch_role_mappings: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -701,7 +701,7 @@ components: required: true security.patch_roles: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -709,7 +709,7 @@ components: required: true security.patch_tenant: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -717,7 +717,7 @@ components: required: true security.patch_tenants: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -725,7 +725,7 @@ components: required: true security.patch_user: content: - application/json: + application/x-ndjson: schema: type: array items: @@ -733,7 +733,7 @@ components: required: true security.patch_users: content: - application/json: + application/x-ndjson: schema: type: array items: diff --git a/tools/helpers.ts b/tools/helpers.ts index 737c9fd84..fe790b8c4 100644 --- a/tools/helpers.ts +++ b/tools/helpers.ts @@ -2,7 +2,7 @@ import fs from 'fs' import YAML from 'yaml' import _ from 'lodash' -export function resolve (ref: string, root: Record) { +export function resolveRef (ref: string, root: Record): Record | undefined{ const paths = ref.replace('#/', '').split('/') for (const p of paths) { root = root[p] @@ -11,6 +11,21 @@ export function resolve (ref: string, root: Record) { return root } +export function resolveObj(obj: Record | undefined, root: Record) { + if (obj === undefined) return undefined + if (obj.$ref) return resolveRef(obj.$ref, root) + return obj +} + +export function dig(obj: Record, path: string[], root: Record): any { + let value = obj + for (const p of path) { + value = resolveObj(value, root)?.[p] + if (value === undefined) break + } + return value +} + export function sortByKey (obj: Record, priorities: string[] = []) { const orders = _.fromPairs(priorities.map((k, i) => [k, i + 1])) const sorted = _.entries(obj).sort((a, b) => { diff --git a/tools/linter/components/NamespaceFile.ts b/tools/linter/components/NamespaceFile.ts index 1a461be9b..0f995e67e 100644 --- a/tools/linter/components/NamespaceFile.ts +++ b/tools/linter/components/NamespaceFile.ts @@ -3,7 +3,7 @@ import { type OperationSpec, type ValidationError } from '../../types' import OperationGroup from './OperationGroup' import _ from 'lodash' import Operation from './Operation' -import { resolve } from '../../helpers' +import { resolveRef } from '../../helpers' import FileValidator from './base/FileValidator' const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'] @@ -68,7 +68,7 @@ export default class NamespaceFile extends FileValidator { validate_unresolved_refs (): ValidationError[] { return Array.from(this.refs()).map((ref) => { - if (resolve(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref) + if (resolveRef(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref) }).filter((e) => e) as ValidationError[] } From fbdf6be1377cddf40614630aab4dca27eaac0ec0 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Tue, 30 Apr 2024 13:17:53 -0600 Subject: [PATCH 2/2] # linting Signed-off-by: Theo Truong --- tools/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/helpers.ts b/tools/helpers.ts index fe790b8c4..dab79201b 100644 --- a/tools/helpers.ts +++ b/tools/helpers.ts @@ -2,7 +2,7 @@ import fs from 'fs' import YAML from 'yaml' import _ from 'lodash' -export function resolveRef (ref: string, root: Record): Record | undefined{ +export function resolveRef (ref: string, root: Record): Record | undefined { const paths = ref.replace('#/', '').split('/') for (const p of paths) { root = root[p] @@ -11,13 +11,13 @@ export function resolveRef (ref: string, root: Record): Record | undefined, root: Record) { +export function resolveObj (obj: Record | undefined, root: Record) { if (obj === undefined) return undefined if (obj.$ref) return resolveRef(obj.$ref, root) return obj } -export function dig(obj: Record, path: string[], root: Record): any { +export function dig (obj: Record, path: string[], root: Record): any { let value = obj for (const p of path) { value = resolveObj(value, root)?.[p]