From b53adb021000519328db8bef1e7e4f64667f559e Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Mon, 26 Aug 2024 16:39:42 +0300 Subject: [PATCH 01/33] Schema validation: update HTTP sample --- .../application_role_http_api/http-api.lua | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_role_http_api/http-api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_role_http_api/http-api.lua index e54d20b45d..b5bb980963 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_role_http_api/http-api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_role_http_api/http-api.lua @@ -1,22 +1,46 @@ -- http-api.lua -- local httpd local json = require('json') +local schema = require('experimental.config.utils.schema') -local function validate(cfg) - if cfg.host then - assert(type(cfg.host) == "string", "'host' should be a string containing a valid IP address") +local function validate_host(host, w) + local host_pattern = "^(%d+)%.(%d+)%.(%d+)%.(%d+)$" + if not host:match(host_pattern) then + w.error("'host' should be a string containing a valid IP address, got %q", host) end - if cfg.port then - assert(type(cfg.port) == "number", "'port' should be a number") - assert(cfg.port >= 1 and cfg.port <= 65535, "'port' should be between 1 and 65535") +end + +local function validate_port(port, w) + if port <= 1 or port >= 65535 then + w.error("'port' should be between 1 and 65535, got %d", port) end end +local listen_address_schema = schema.new('listen_address', schema.record({ + host = schema.scalar({ + type = 'string', + validate = validate_host, + default = '127.0.0.1', + }), + port = schema.scalar({ + type = 'integer', + validate = validate_port, + default = 8080, + }), +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + local function apply(cfg) if httpd then httpd:stop() end - httpd = require('http.server').new(cfg.host, cfg.port) + local cfg_with_defaults = listen_address_schema:apply_default(cfg) + local host = listen_address_schema:get(cfg_with_defaults, 'host') + local port = listen_address_schema:get(cfg_with_defaults, 'port') + httpd = require('http.server').new(host, port) local response_headers = { ['content-type'] = 'application/json' } httpd:route({ path = '/band/:id', method = 'GET' }, function(req) local id = req:stash('id') From baddf54f955951271c2ee2a879f5e24a5a09dfd5 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Mon, 26 Aug 2024 15:56:20 +0300 Subject: [PATCH 02/33] Schema validation: samples --- .../config.yaml | 15 +++++ .../http-api.lua | 34 +++++++++++ .../instances.yml | 1 + .../config.yaml | 12 ++++ .../http_api.lua | 30 ++++++++++ .../instances.yml | 1 + .../config.yaml | 12 ++++ .../http_api.lua | 56 +++++++++++++++++++ .../instances.yml | 1 + .../api.lua | 51 +++++++++++++++++ .../config.yaml | 14 +++++ .../instances.yml | 1 + .../config.yaml | 12 ++++ .../http_api.lua | 41 ++++++++++++++ .../instances.yml | 1 + .../config.yaml | 15 +++++ .../http_api.lua | 38 +++++++++++++ .../instances.yml | 1 + .../config.yaml | 12 ++++ .../http_api.lua | 35 ++++++++++++ .../instances.yml | 1 + .../config.yaml | 9 +++ .../http_api.lua | 24 ++++++++ .../instances.yml | 1 + 24 files changed, 418 insertions(+) create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml new file mode 100644 index 0000000000..2b49dc5420 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml @@ -0,0 +1,15 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http-api ] + roles_cfg: + http-api: + - host: '127.0.0.1' + port: 8080 + scheme: 'http' + - host: '127.0.0.1' + port: 8443 + scheme: 'https' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua new file mode 100644 index 0000000000..04ab782c92 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua @@ -0,0 +1,34 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local listen_address_schema = schema.new('listen_address', schema.array({ + items = schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }) + }) +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + for _, uri in pairs(cfg) do + local scheme = uri.scheme + local host = uri.host + local port = uri.port + log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) + end +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml new file mode 100644 index 0000000000..bd2f31e7f4 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml @@ -0,0 +1,12 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua new file mode 100644 index 0000000000..46e3f13933 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua @@ -0,0 +1,30 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local listen_address_schema = schema.new('listen_address', schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }) +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = listen_address_schema:get(cfg, 'scheme') + local host = listen_address_schema:get(cfg, 'host') + local port = listen_address_schema:get(cfg, 'port') + log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml new file mode 100644 index 0000000000..bd2f31e7f4 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml @@ -0,0 +1,12 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua new file mode 100644 index 0000000000..5619769d6e --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua @@ -0,0 +1,56 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local function validate_host(host, w) + local host_pattern = "^(%d+)%.(%d+)%.(%d+)%.(%d+)$" + if not host:match(host_pattern) then + w.error("'host' should be a string containing a valid IP address, got %q", host) + end +end + +local function validate_port(port, w) + if port <= 1 or port >= 65535 then + w.error("'port' should be between 1 and 65535, got %d", port) + end +end + +local listen_address_schema = schema.new('listen_address', schema.record({ + scheme = schema.scalar({ + type = 'string', + allowed_values = { 'http', 'https' }, + default = 'http', + }), + host = schema.scalar({ + type = 'string', + validate = validate_host, + default = '127.0.0.1', + }), + port = schema.scalar({ + type = 'integer', + validate = validate_port, + default = 8080, + }), +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local cfg_with_defaults = listen_address_schema:apply_default(cfg) + local scheme = listen_address_schema:get(cfg_with_defaults, 'scheme') + local host = listen_address_schema:get(cfg_with_defaults, 'host') + local port = listen_address_schema:get(cfg_with_defaults, 'port') + log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua new file mode 100644 index 0000000000..a8c5de32c4 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua @@ -0,0 +1,51 @@ +-- api.lua -- +local log = require('log').new("api") +local schema = require('experimental.config.utils.schema') + +local listen_address = schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }) +}, { + validate = function(data, w) + local protocol = w.schema.computed.annotations.protocol + if protocol == 'grpc' and data.scheme then + w.error("grpc doesn't support 'scheme'") + end + end, +}) + +local http_listen_address_schema = schema.new('http_listen_address', schema.record({ + name = schema.scalar({ type = 'string' }), + listen_address = listen_address, +}, { + protocol = 'http', +})) + +local grpc_listen_address_schema = schema.new('grpc_listen_address', schema.record({ + name = schema.scalar({ type = 'string' }), + listen_address = listen_address, +}, { + protocol = 'grpc', +})) + +local function validate(cfg) + http_listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = http_listen_address_schema:get(cfg, 'listen_address.scheme') + local host = http_listen_address_schema:get(cfg, 'listen_address.host') + local port = http_listen_address_schema:get(cfg, 'listen_address.port') + log.info("API endpoint: %s://%s:%d", scheme, host, port) +end + +local function stop() + log.info("The 'api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml new file mode 100644 index 0000000000..01db397f76 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml @@ -0,0 +1,14 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ api ] + roles_cfg: + api: + name: 'HTTP listen address' + listen_address: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml new file mode 100644 index 0000000000..bd2f31e7f4 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml @@ -0,0 +1,12 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua new file mode 100644 index 0000000000..ede8f6b1d1 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua @@ -0,0 +1,41 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local scheme_node = schema.enum({ 'http', 'https' }) +local host_node = schema.scalar({ type = 'string' }) +local port_node = schema.scalar({ type = 'integer' }) + +local scheme_from_env = schema.fromenv('HTTP_SCHEME', os.getenv('HTTP_SCHEME'), scheme_node) +local host_from_env = schema.fromenv('HTTP_HOST', os.getenv('HTTP_HOST'), host_node) +local port_from_env = schema.fromenv('HTTP_PORT', os.getenv('HTTP_PORT'), port_node) + +local listen_address_schema = schema.new('listen_address', schema.record({ + scheme = scheme_node, + host = host_node, + port = port_node +})) + +local function validate(cfg) + listen_address_schema:set(cfg, 'scheme', scheme_from_env) + listen_address_schema:set(cfg, 'host', host_from_env) + listen_address_schema:set(cfg, 'port', port_from_env) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = listen_address_schema:get(cfg, 'scheme') + local host = listen_address_schema:get(cfg, 'host') + local port = listen_address_schema:get(cfg, 'port') + log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml new file mode 100644 index 0000000000..5eb4de13ce --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml @@ -0,0 +1,15 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + host: '127.0.0.1' + port: 8080 + scheme: 'http' + query_params: + skip: 0 + limit: 10 diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua new file mode 100644 index 0000000000..1ffb679772 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua @@ -0,0 +1,38 @@ +-- http_api.lua -- +log = require('log').new("http_api") +schema = require('experimental.config.utils.schema') + +listen_address_schema = schema.new('listen_address', schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }), + query_params = schema.map({ key = schema.scalar({ type = 'string' }), + value = schema.scalar({ type = 'integer' }) }) +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = listen_address_schema:get(cfg, 'scheme') + local host = listen_address_schema:get(cfg, 'host') + local port = listen_address_schema:get(cfg, 'port') + local query_params = listen_address_schema:get(cfg, 'query_params') + local query_string = '' + for name, value in pairs(query_params) do + query_string = query_string .. name .. '=' .. value .. '&' + end + local query_string_without_amp = string.gsub(query_string, "&$", "") + log.info("HTTP API endpoint: %s://%s:%d?%s", scheme, host, port, query_string_without_amp) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml new file mode 100644 index 0000000000..bd2f31e7f4 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml @@ -0,0 +1,12 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua new file mode 100644 index 0000000000..e60caf0452 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua @@ -0,0 +1,35 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local listen_address_schema = schema.new('listen_address', schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }) +}), { methods = { + get_url_string = function(_self, url) + return string.format("%s://%s:%d", url.scheme, url.host, url.port) + end, +} }) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = listen_address_schema:get(cfg, 'scheme') + local host = listen_address_schema:get(cfg, 'host') + local port = listen_address_schema:get(cfg, 'port') + local url_string = listen_address_schema:get_url_string({ scheme = scheme, host = host, port = port }) + log.info("HTTP API endpoint: %s", url_string) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml @@ -0,0 +1 @@ +instance001: diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml new file mode 100644 index 0000000000..2d56515801 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml @@ -0,0 +1,9 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: 'http://127.0.0.1:8080' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua new file mode 100644 index 0000000000..c8aea6e018 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua @@ -0,0 +1,24 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local http_api_schema = schema.new('http_api', schema.scalar({ type = 'string' })) + +local function validate(cfg) + http_api_schema:validate(cfg) +end + +local function apply(cfg) + local http_api_cfg = http_api_schema:get(cfg) + log.info("HTTP API endpoint: %s", http_api_cfg) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml @@ -0,0 +1 @@ +instance001: From f1261a6ba38a6461e174ea907dee8cc36a07050d Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 14:52:14 +0300 Subject: [PATCH 03/33] Schema validation: samples - use endpoints instead of query params --- .../config.yaml | 7 ++++--- .../http_api.lua | 15 +++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml index 5eb4de13ce..55edfabbb5 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml @@ -10,6 +10,7 @@ groups: host: '127.0.0.1' port: 8080 scheme: 'http' - query_params: - skip: 0 - limit: 10 + endpoints: + user: true + order: true + customer: false diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua index 1ffb679772..4535c88e68 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua @@ -6,8 +6,8 @@ listen_address_schema = schema.new('listen_address', schema.record({ scheme = schema.enum({ 'http', 'https' }), host = schema.scalar({ type = 'string' }), port = schema.scalar({ type = 'integer' }), - query_params = schema.map({ key = schema.scalar({ type = 'string' }), - value = schema.scalar({ type = 'integer' }) }) + endpoints = schema.map({ key = schema.scalar({ type = 'string' }), + value = schema.scalar({ type = 'boolean' }) }) })) local function validate(cfg) @@ -18,13 +18,12 @@ local function apply(cfg) local scheme = listen_address_schema:get(cfg, 'scheme') local host = listen_address_schema:get(cfg, 'host') local port = listen_address_schema:get(cfg, 'port') - local query_params = listen_address_schema:get(cfg, 'query_params') - local query_string = '' - for name, value in pairs(query_params) do - query_string = query_string .. name .. '=' .. value .. '&' + local endpoints = listen_address_schema:get(cfg, 'endpoints') + for name, value in pairs(endpoints) do + if value then + log.info("HTTP API endpoint: %s://%s:%d/%s", scheme, host, port, name) + end end - local query_string_without_amp = string.gsub(query_string, "&$", "") - log.info("HTTP API endpoint: %s://%s:%d?%s", scheme, host, port, query_string_without_amp) end local function stop() From e4df6d8c54783a9c5fe4a91ec83e1b5e65c5d1e3 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 15:06:52 +0300 Subject: [PATCH 04/33] Schema validation: samples - formatting/naming/cfg --- .../http_api.lua | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua index e60caf0452..0ab1ce4bfe 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua @@ -2,25 +2,30 @@ local log = require('log').new("http_api") local schema = require('experimental.config.utils.schema') -local listen_address_schema = schema.new('listen_address', schema.record({ - scheme = schema.enum({ 'http', 'https' }), - host = schema.scalar({ type = 'string' }), - port = schema.scalar({ type = 'integer' }) -}), { methods = { - get_url_string = function(_self, url) - return string.format("%s://%s:%d", url.scheme, url.host, url.port) - end, -} }) +local listen_address_schema = schema.new( + "listen_address", + schema.record( + { + scheme = schema.enum({ "http", "https" }), + host = schema.scalar({ type = "string" }), + port = schema.scalar({ type = "integer" }) + } + ), + { + methods = { + format = function(_self, url) + return string.format("%s://%s:%d", url.scheme, url.host, url.port) + end + } + } +) local function validate(cfg) listen_address_schema:validate(cfg) end local function apply(cfg) - local scheme = listen_address_schema:get(cfg, 'scheme') - local host = listen_address_schema:get(cfg, 'host') - local port = listen_address_schema:get(cfg, 'port') - local url_string = listen_address_schema:get_url_string({ scheme = scheme, host = host, port = port }) + local url_string = listen_address_schema:format(cfg) log.info("HTTP API endpoint: %s", url_string) end From c180f394cb619cbfda64d5cea9ec77639093757e Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 15:10:06 +0300 Subject: [PATCH 05/33] Schema validation: samples - use ipairs --- .../application_validate_cfg_array_record/http-api.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua index 04ab782c92..ae7cb6ccf7 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua @@ -15,7 +15,7 @@ local function validate(cfg) end local function apply(cfg) - for _, uri in pairs(cfg) do + for _, uri in ipairs(cfg) do local scheme = uri.scheme local host = uri.host local port = uri.port From 55671657177c8c3b3a0b9c8962bcee8e1997f532 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 15:26:29 +0300 Subject: [PATCH 06/33] Schema validation: samples - drop grpc/check nil --- .../api.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua index a8c5de32c4..f73e2f3bcc 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua @@ -9,8 +9,8 @@ local listen_address = schema.record({ }, { validate = function(data, w) local protocol = w.schema.computed.annotations.protocol - if protocol == 'grpc' and data.scheme then - w.error("grpc doesn't support 'scheme'") + if protocol == 'iproto' and data.scheme ~= nil then + w.error("iproto doesn't support 'scheme'") end end, }) @@ -22,11 +22,11 @@ local http_listen_address_schema = schema.new('http_listen_address', schema.reco protocol = 'http', })) -local grpc_listen_address_schema = schema.new('grpc_listen_address', schema.record({ +local iproto_listen_address_schema = schema.new('iproto_listen_address', schema.record({ name = schema.scalar({ type = 'string' }), listen_address = listen_address, }, { - protocol = 'grpc', + protocol = 'iproto', })) local function validate(cfg) From 8330c478ac07e05cfca5c0a0e2a933212e410212 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 16:14:40 +0300 Subject: [PATCH 07/33] Schema validation: samples - update env sample --- .../http_api.lua | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua index ede8f6b1d1..481603c893 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua @@ -2,32 +2,34 @@ local log = require('log').new("http_api") local schema = require('experimental.config.utils.schema') -local scheme_node = schema.enum({ 'http', 'https' }) -local host_node = schema.scalar({ type = 'string' }) -local port_node = schema.scalar({ type = 'integer' }) - -local scheme_from_env = schema.fromenv('HTTP_SCHEME', os.getenv('HTTP_SCHEME'), scheme_node) -local host_from_env = schema.fromenv('HTTP_HOST', os.getenv('HTTP_HOST'), host_node) -local port_from_env = schema.fromenv('HTTP_PORT', os.getenv('HTTP_PORT'), port_node) - local listen_address_schema = schema.new('listen_address', schema.record({ - scheme = scheme_node, - host = host_node, - port = port_node + scheme = schema.enum({ 'http', 'https' }, { env = 'HTTP_SCHEME' }), + host = schema.scalar({ type = 'string', env = 'HTTP_HOST' }), + port = schema.scalar({ type = 'integer', env = 'HTTP_PORT' }) })) +local function collect_env_cfg() + local res = {} + for _, w in listen_address_schema:pairs() do + local env_var = w.schema.env + if env_var ~= nil then + local value = schema.fromenv(env_var, os.getenv(env_var), w.schema) + listen_address_schema:set(res, w.path, value) + end + end + return res +end + local function validate(cfg) - listen_address_schema:set(cfg, 'scheme', scheme_from_env) - listen_address_schema:set(cfg, 'host', host_from_env) - listen_address_schema:set(cfg, 'port', port_from_env) - listen_address_schema:validate(cfg) + local env_cfg = collect_env_cfg() + local result_cfg = listen_address_schema:merge(cfg, env_cfg) + listen_address_schema:validate(result_cfg) end local function apply(cfg) - local scheme = listen_address_schema:get(cfg, 'scheme') - local host = listen_address_schema:get(cfg, 'host') - local port = listen_address_schema:get(cfg, 'port') - log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) + local env_cfg = collect_env_cfg() + local result_cfg = listen_address_schema:merge(cfg, env_cfg) + log.info("HTTP API endpoint: %s://%s:%d", result_cfg.scheme, result_cfg.host, result_cfg.port) end local function stop() From d28c40a9921cc95e3765dd451b38a92dc2db0669 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Fri, 30 Aug 2024 17:06:01 +0300 Subject: [PATCH 08/33] Schema validation: samples - add local --- .../application_validate_cfg_record_map/http_api.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua index 4535c88e68..99c52e488b 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua @@ -1,8 +1,8 @@ -- http_api.lua -- -log = require('log').new("http_api") -schema = require('experimental.config.utils.schema') +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') -listen_address_schema = schema.new('listen_address', schema.record({ +local listen_address_schema = schema.new('listen_address', schema.record({ scheme = schema.enum({ 'http', 'https' }), host = schema.scalar({ type = 'string' }), port = schema.scalar({ type = 'integer' }), From 31f3df30cd41929c1cef123fb04d1790ed334df0 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Mon, 26 Aug 2024 14:45:19 +0300 Subject: [PATCH 09/33] Schema validation --- doc/reference/reference_lua/config.rst | 6 + .../reference_lua/config/utils_schema.rst | 814 ++++++++++++++++++ 2 files changed, 820 insertions(+) create mode 100644 doc/reference/reference_lua/config/utils_schema.rst diff --git a/doc/reference/reference_lua/config.rst b/doc/reference/reference_lua/config.rst index fd6ce07533..6039e9c0c7 100644 --- a/doc/reference/reference_lua/config.rst +++ b/doc/reference/reference_lua/config.rst @@ -523,3 +523,9 @@ The ``config.storage`` API allows you to interact with a Tarantool-based :ref:`c :dedent: Example on GitHub: `tarantool_config_storage `_ + +.. toctree:: + :maxdepth: 1 + :includehidden: + + config/utils_schema diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst new file mode 100644 index 0000000000..ecf617e05d --- /dev/null +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -0,0 +1,814 @@ +.. _config_utils_schema_module: + +Submodule experimental.config.utils.schema +========================================== + +**Since:** :doc:`3.2.0 ` + +Tarantool allows you to provide arbitrary configurations for cluster applications: + +- For applications loaded using the :ref:`app ` option, :ref:`app.cfg ` is used to provide a configuration. +- For :ref:`custom roles ` developed as a part of a cluster application, :ref:`roles_cfg ` is used. + +The ``experimental.config.utils.schema`` module can be used to validate such configurations and process their data: get and set configuration values, filter and transform configuration data, and so on. + +.. important:: + + ``experimental.config.utils.schema`` is an experimental submodule and is subject to changes. + + + + +.. _config_utils_schema_getting_started: + +Getting started with config.utils.schema +---------------------------------------- + +Example config - scalar type: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml + :language: yaml + :start-at: roles + :dedent: + +1. Load the module: + + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: config.utils.schema + :end-at: config.utils.schema + :dedent: + +2. Define a schema using :ref:`config.utils.schema.new() `. + Example enum: + + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local http_api_schema + :end-before: local function validate + :dedent: + +3. Validate config values using ``config.utils.schema.validate()``. + For a role, inside the ``validate()`` func: + + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local function validate + :end-before: local function apply + :dedent: + +4. Get value. For a role, inside the ``apply()`` func: + + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local function apply + :end-before: local function stop + :dedent: + + + +.. _config_utils_schema_definition: + +Defining a schema +----------------- + +Create using :ref:`config.utils.schema.new() `. Options: + +- schema name +- schema node: + + - scalar and composite (record, array, map) types + - annotations (type, validate, and so on) + +- (optional) methods + +.. _config_utils_schema_nodes: + +Schema nodes +~~~~~~~~~~~~ + +.. _config_utils_schema_scalar_composite_types: + +Scalar and composite types +************************** + +There are scalar and composite types. + +- :ref:`config.utils.schema.scalar() ` +- :ref:`config.utils.schema.record() ` +- :ref:`config.utils.schema.map() ` +- :ref:`config.utils.schema.array() ` + +Shortcuts: + +- :ref:`config.utils.schema.enum() ` +- :ref:`config.utils.schema.set() ` + +.. _config_utils_schema_type_system_scalar: + +Scalar +^^^^^^ + +:ref:`config.utils.schema.scalar() ` + +Example config - scalar type: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema definition: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local http_api_schema + :end-before: local function validate + :dedent: + +See also: :ref:`config_utils_schema_data_types`. + + + +.. _record: + +Record +^^^^^^ + +Example config: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + +.. _array_record: + +Array +^^^^^ + +Also: :ref:`config.utils.schema.set() ` + +Example config: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + +.. _record_map: + +Map +^^^ + +Example config: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + + + + +.. _config_utils_schema_annotation: + +Annotations +*********** + +- ``type`` (string) +- ``validate`` (function) +- ``allowed_values`` (table) +- ``default`` (any) +- ``apply_default_if`` (function) +- custom annotations + +.. _config_utils_schema_data_types: + +Type annotation +^^^^^^^^^^^^^^^ + +Example config - scalar type: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema definition: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local http_api_schema + :end-before: local function validate + :dedent: + +Supported types: + +- ``string`` -- ``string`` +- ``number`` -- ``number`` +- ``integer`` -- ``number`` +- ``boolean`` -- ``boolean`` +- ``any`` - any accepts an arbitrary Lua type, including ``table``. A scalar of the ``any`` type may be used to declare an arbitrary value that doesn't need any validation. +- ``string, number`` -- ``string`` or ``number`` +- ``number, string`` -- ``string`` or ``string`` + + +.. _config_utils_schema_validate_annotation: + +Validate annotation +^^^^^^^^^^^^^^^^^^^ + +Example config: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema definition (``validate``, ``allowed_values``, ``default``): + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + +Validate functions: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua + :language: lua + :start-at: local function validate_host + :end-before: local listen_address_schema + :dedent: + + + + +.. _config_utils_schema_custom_annotations: + +Custom annotations +^^^^^^^^^^^^^^^^^^ + +``env``: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function collect_env_cfg + :dedent: + +Learn more about getting env vars: :ref:`config_utils_schema_env-vars`. + + +.. _config_utils_schema_computed_annotations: + +Computed annotations +^^^^^^^^^^^^^^^^^^^^ + +Schema node: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua + :language: lua + :start-at: local listen_address + :end-before: local http_listen_address_schema + :dedent: + +Valid schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua + :language: lua + :start-at: local http_listen_address_schema + :end-before: local iproto_listen_address_schema + :dedent: + +Invalid schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua + :language: lua + :start-at: local iproto_listen_address_schema + :end-before: local function validate + :dedent: + + + +.. _config_utils_schema_methods: + +User-defined methods +~~~~~~~~~~~~~~~~~~~~ + +Example config: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + + + + + +.. _config_utils_schema_process_data: + +Processing configuration data +----------------------------- + +.. _config_utils_schema_validating_configuration: + +Validating configuration +~~~~~~~~~~~~~~~~~~~~~~~~ + +Role -- inside the ``validate()`` function: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local function validate + :end-before: local function apply + :dedent: + + +.. _config_utils_schema_get_configuration: + +Getting configuration values +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Role -- inside the ``apply()`` function (scalar): + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + :language: lua + :start-at: local function apply + :end-before: local function stop + :dedent: + +Record -- pass field names: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua + :language: lua + :start-at: local function apply + :end-before: local function stop + :dedent: + +Array: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua + :language: lua + :start-at: local function apply + :end-before: local function stop + :dedent: + + + +.. _config_utils_schema_env-vars: + +Parsing environment variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using :ref:`config.utils.schema.fromenv() `: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + +Other API members: + +- Custom annotation (``env``) +- ``pairs()`` +- ``set()`` + + + +.. _api-reference-config-utils-schema: + +API Reference +------------- + +.. container:: table + + .. rst-class:: left-align-column-1 + .. rst-class:: left-align-column-2 + + .. list-table:: + :widths: 45 55 + + * - **Functions** + - + + * - :ref:`config.utils.schema.array() ` + - TODO + + * - :ref:`config.utils.schema.enum() ` + - TODO + + * - :ref:`config.utils.schema.fromenv() ` + - TODO + + * - :ref:`config.utils.schema.map() ` + - TODO + + * - :ref:`config.utils.schema.new() ` + - TODO + + * - :ref:`config.utils.schema.record() ` + - TODO + + * - :ref:`config.utils.schema.scalar() ` + - TODO + + * - :ref:`config.utils.schema.set() ` + - TODO + + * - **schema_object** + - + + * - :ref:`schema_object:apply_default() ` + - TODO + + * - :ref:`schema_object:filter() ` + - TODO + + * - :ref:`schema_object:get() ` + - TODO + + * - :ref:`schema_object:map() ` + - TODO + + * - :ref:`schema_object.methods ` + - TODO + + * - :ref:`schema_object.name ` + - TODO + + * - :ref:`schema_object:merge() ` + - TODO + + * - :ref:`schema_object:pairs() ` + - TODO + + * - :ref:`schema_object:set() ` + - TODO + + * - :ref:`schema_object.schema ` + - TODO + + * - :ref:`schema_object:validate() ` + - TODO + + * - **schema_node_annotation** + - + + * - :ref:`schema_node_annotation.allowed_values ` + - TODO + + * - :ref:`schema_node_annotation.apply_default_if ` + - TODO + + * - :ref:`schema_node_annotation.default ` + - TODO + + * - :ref:`schema_node_annotation.type ` + - TODO + + * - :ref:`schema_node_annotation.validate ` + - TODO + + * - **schema_node_object** + - + + * - :ref:`schema_node_object.computed.annotations ` + - TODO + + * - :ref:`schema_node_object.fields ` + - TODO + + * - :ref:`schema_node_object.type ` + - TODO + + + + + + + + +.. _config-utils-schema_functions: + +Functions +~~~~~~~~~ + +.. module:: config.utils.schema + +.. _config-utils-schema-array: + +.. function:: schema.array(opts) + + Define an array. + + :param table opts: a table in the following format: + + .. code-block:: text + + { items = , <..annotations..> } + + See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + +.. _config-utils-schema-enum: + +.. function:: schema.enum(allowed_values, annotations) + + Create an enum. + + :param table allowed_values: allowed values + :param table annotations: annotations (see :ref:`schema_node_annotation `) + +.. _config-utils-schema-fromenv: + +.. function:: schema.fromenv(env_var_name, raw_value, schema_node) + + Parse data from an environment variable as a value of the given type. + + :param string env_var_name: env var name + :param string raw_value: raw value + :param schema_node_object schema_node: a schema node (see :ref:`schema_node_object `) + + +.. _config-utils-schema-map: + +.. function:: schema.map(opts) + + Create a map. + + :param table opts: a table in the following format: + + .. code-block:: text + + { key = , value = , <..annotations..> } + + See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + + :return: a table that represents the created schema node + :rtype: table + +.. _config-utils-schema-new: + +.. function:: schema.new(schema_name, schema_node[, { methods = <...> }]) + + Create a schema object. + + :param string schema_name: a name + :param table schema_node: a node + :param table methods: methods + + :return: a new schema instance (see :ref:`schema_object `) + :rtype: userdata + + +.. _config-utils-schema-record: + +.. function:: schema.record(fields[, annotations]) + + Create a record. + + :param table fields: a table of fields in the following format: + + .. code-block:: text + + { [] = , <...> } + + See also: :ref:`schema_node_object `. + + :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) + + :return: a table that represents the created schema node + :rtype: table + + +.. _config-utils-schema-scalar: + +.. function:: schema.scalar(type[, annotations]) + + Create a scalar. + + :param string type: data type (see :ref:`config_utils_schema_data_types`) + :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) + + :return: a table that represents the created schema node + :rtype: table + +.. _config-utils-schema-set: + +.. function:: schema.set(allowed_values, annotations) + + Shortcut for array of unique string values from the given list of allowed values. + + :param table allowed_values: allowed values + :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) + + :return: a table that represents the created schema node + :rtype: table + + +.. _config-utils-schema_object: + +schema_object +~~~~~~~~~~~~~ + +.. class:: schema_object + + .. _config-schema_object-apply_default: + + .. method:: apply_default(data) + + Apply default values. + + :param any data: data + + :return: new data + + .. _config-schema_object-filter: + + .. method:: filter(data, f) + + Filter data based on the schema annotations. + + :return: a luafun iterator + + .. _config-schema_object-get: + + .. method:: get(data, path) + + Get nested data that is pointed by the given path. + + Example: see :ref:`config_utils_schema_get_configuration` + + .. _config-schema_object-map: + + .. method:: map(data, f, f_ctx) + + Transform data by the given function. + + :param any data: value at the given path + :param any f: walkthrough node, described below + :param any f_ctx: user-provided context for the transformation function + + .. _config-schema_object-methods: + + .. method:: methods() + + TODO + + .. _config-schema_object-name: + + .. data:: name + + TODO + + .. _config-schema_object-merge: + + .. method:: merge(a, b) + + Merge two hierarchical values (prefer the latter). + + :param any a: data + :param any b: data + + :return: new data + + .. _config-schema_object-pairs: + + .. method:: pairs() + + Traverse the schema. + + :return: a luafun iterator + + .. _config-schema_object-set: + + .. method:: set() + + TODO + + .. _config-schema_object-schema: + + .. data:: schema + + TODO + + .. _config-schema_object-validate: + + .. method:: validate(data) + + Validate data against the schema. + ``validate()`` raises an error if the specified data doesn't adhere this schema. + + :param any data: data + + Example: see :ref:`config_utils_schema_validate_annotation` + + + + + +.. _config-utils-schema_node_annotation: + +schema_node_annotation +~~~~~~~~~~~~~~~~~~~~~~ + +.. class:: schema_node_annotation + + .. _config-schema_node_annotation-allowed_values: + + .. data:: allowed_values + + A list of allowed values. + + Example: see :ref:`config_utils_schema_validate_annotation` + + .. _config-schema_node_annotation-apply_default_if: + + .. data:: apply_default_if + + A function that specifies whether to apply the default value specified using ``default``. + + .. _config-schema_node_annotation-default: + + .. data:: default + + The value to be placed instead of a missed one. + + Example: see :ref:`config_utils_schema_validate_annotation` + + .. _config-schema_node_annotation-type: + + .. data:: type + + A value type. See :ref:`config_utils_schema_data_types`. + + Example: see :ref:`config_utils_schema_validate_annotation` + + .. _config-schema_node_annotation-validate: + + .. data:: validate + + A function used to validate data. + + Example: see :ref:`config_utils_schema_validate_annotation` + + +.. _config-utils-schema_node_object: + +schema_node_object +~~~~~~~~~~~~~~~~~~ + +.. class:: schema_node_object + + .. _config-schema_node_object-computed-annotations: + + .. data:: schema_node_object.computed.annotations + + TODO + + .. _config-schema_node_object-fields: + + .. data:: fields + + TODO + + .. _config-schema_node_object-type: + + .. data:: type + + TODO From ac1177bb4b140865ccab35dad34be24bd83cf7fb Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Wed, 11 Sep 2024 18:37:03 +0300 Subject: [PATCH 10/33] Schema validation - review --- .../reference_lua/config/utils_schema.rst | 204 ++++++++---------- 1 file changed, 92 insertions(+), 112 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index ecf617e05d..2a5c396010 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -39,7 +39,7 @@ Example config - scalar type: :end-at: config.utils.schema :dedent: -2. Define a schema using :ref:`config.utils.schema.new() `. +2. Define a schema using :ref:`schema.new() `. Example enum: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua @@ -72,7 +72,7 @@ Example config - scalar type: Defining a schema ----------------- -Create using :ref:`config.utils.schema.new() `. Options: +Create using :ref:`schema.new() `. Options: - schema name - schema node: @@ -94,22 +94,22 @@ Scalar and composite types There are scalar and composite types. -- :ref:`config.utils.schema.scalar() ` -- :ref:`config.utils.schema.record() ` -- :ref:`config.utils.schema.map() ` -- :ref:`config.utils.schema.array() ` +- :ref:`schema.scalar() ` +- :ref:`schema.record() ` +- :ref:`schema.map() ` +- :ref:`schema.array() ` Shortcuts: -- :ref:`config.utils.schema.enum() ` -- :ref:`config.utils.schema.set() ` +- :ref:`schema.enum() ` +- :ref:`schema.set() ` .. _config_utils_schema_type_system_scalar: Scalar ^^^^^^ -:ref:`config.utils.schema.scalar() ` +:ref:`schema.scalar() ` Example config - scalar type: @@ -150,12 +150,17 @@ Schema: :end-before: local function validate :dedent: + +TODO: add a sample with nested records. It can be used in the ``Processing configuration data`` section. + + + .. _array_record: Array ^^^^^ -Also: :ref:`config.utils.schema.set() ` +Also: :ref:`schema.set() ` Example config: @@ -193,39 +198,10 @@ Schema: :dedent: - - -.. _config_utils_schema_annotation: - -Annotations -*********** - -- ``type`` (string) -- ``validate`` (function) -- ``allowed_values`` (table) -- ``default`` (any) -- ``apply_default_if`` (function) -- custom annotations - .. _config_utils_schema_data_types: -Type annotation -^^^^^^^^^^^^^^^ - -Example config - scalar type: - -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml - :language: yaml - :start-at: roles - :dedent: - -Schema definition: - -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua - :language: lua - :start-at: local http_api_schema - :end-before: local function validate - :dedent: +Data types +********** Supported types: @@ -238,10 +214,12 @@ Supported types: - ``number, string`` -- ``string`` or ``string`` -.. _config_utils_schema_validate_annotation: -Validate annotation -^^^^^^^^^^^^^^^^^^^ + +.. _config_utils_schema_annotation: + +Annotations +*********** Example config: @@ -269,12 +247,12 @@ Validate functions: -.. _config_utils_schema_custom_annotations: +.. _config_utils_schema_user_defined_annotations: -Custom annotations -^^^^^^^^^^^^^^^^^^ +User-defined annotations +^^^^^^^^^^^^^^^^^^^^^^^^ -``env``: +The ``env`` annotation: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua :language: lua @@ -282,7 +260,7 @@ Custom annotations :end-before: local function collect_env_cfg :dedent: -Learn more about getting env vars: :ref:`config_utils_schema_env-vars`. +See the full sample here: :ref:`config_utils_schema_env-vars`. .. _config_utils_schema_computed_annotations: @@ -298,7 +276,7 @@ Schema node: :end-before: local http_listen_address_schema :dedent: -Valid schema: +Passes validation: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua :language: lua @@ -306,7 +284,7 @@ Valid schema: :end-before: local iproto_listen_address_schema :dedent: -Invalid schema: +Raises an error: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua :language: lua @@ -389,13 +367,21 @@ Array: :dedent: +.. _config_utils_schema_transform_configuration: + +Transforming configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TODO: filter, merge, map, apply_default ? + + .. _config_utils_schema_env-vars: Parsing environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Using :ref:`config.utils.schema.fromenv() `: +Using :ref:`schema.fromenv() `: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua :language: lua @@ -427,28 +413,28 @@ API Reference * - **Functions** - - * - :ref:`config.utils.schema.array() ` + * - :ref:`schema.array() ` - TODO - * - :ref:`config.utils.schema.enum() ` + * - :ref:`schema.enum() ` - TODO - * - :ref:`config.utils.schema.fromenv() ` + * - :ref:`schema.fromenv() ` - TODO - * - :ref:`config.utils.schema.map() ` + * - :ref:`schema.map() ` - TODO - * - :ref:`config.utils.schema.new() ` + * - :ref:`schema.new() ` - TODO - * - :ref:`config.utils.schema.record() ` + * - :ref:`schema.record() ` - TODO - * - :ref:`config.utils.schema.scalar() ` + * - :ref:`schema.scalar() ` - TODO - * - :ref:`config.utils.schema.set() ` + * - :ref:`schema.set() ` - TODO * - **schema_object** @@ -466,11 +452,7 @@ API Reference * - :ref:`schema_object:map() ` - TODO - * - :ref:`schema_object.methods ` - - TODO - * - :ref:`schema_object.name ` - - TODO * - :ref:`schema_object:merge() ` - TODO @@ -481,28 +463,34 @@ API Reference * - :ref:`schema_object:set() ` - TODO - * - :ref:`schema_object.schema ` + * - :ref:`schema_object:validate() ` - TODO - * - :ref:`schema_object:validate() ` + * - :ref:`schema_object.methods ` + - TODO + + * - :ref:`schema_object.name ` + - TODO + + * - :ref:`schema_object.schema ` - TODO * - **schema_node_annotation** - - * - :ref:`schema_node_annotation.allowed_values ` + * - :ref:`allowed_values ` - TODO - * - :ref:`schema_node_annotation.apply_default_if ` + * - :ref:`apply_default_if ` - TODO - * - :ref:`schema_node_annotation.default ` + * - :ref:`default ` - TODO - * - :ref:`schema_node_annotation.type ` + * - :ref:`type ` - TODO - * - :ref:`schema_node_annotation.validate ` + * - :ref:`validate ` - TODO * - **schema_node_object** @@ -684,18 +672,6 @@ schema_object :param any f: walkthrough node, described below :param any f_ctx: user-provided context for the transformation function - .. _config-schema_object-methods: - - .. method:: methods() - - TODO - - .. _config-schema_object-name: - - .. data:: name - - TODO - .. _config-schema_object-merge: .. method:: merge(a, b) @@ -721,11 +697,7 @@ schema_object TODO - .. _config-schema_object-schema: - - .. data:: schema - - TODO + .. _config-schema_object-methods: .. _config-schema_object-validate: @@ -736,56 +708,62 @@ schema_object :param any data: data - Example: see :ref:`config_utils_schema_validate_annotation` + Example: see :ref:`config_utils_schema_annotation` + .. data:: methods + TODO + .. _config-schema_object-name: + .. data:: name -.. _config-utils-schema_node_annotation: + TODO + + .. _config-schema_object-schema: + + .. data:: schema + + TODO -schema_node_annotation -~~~~~~~~~~~~~~~~~~~~~~ -.. class:: schema_node_annotation - .. _config-schema_node_annotation-allowed_values: - .. data:: allowed_values - A list of allowed values. +.. _config-utils-schema_node_annotation: - Example: see :ref:`config_utils_schema_validate_annotation` +schema_node_annotation +~~~~~~~~~~~~~~~~~~~~~~ - .. _config-schema_node_annotation-apply_default_if: +.. _config-schema_node_annotation-allowed_values: - .. data:: apply_default_if +- ``allowed_values`` - A function that specifies whether to apply the default value specified using ``default``. + A list of allowed values. - .. _config-schema_node_annotation-default: +.. _config-schema_node_annotation-apply_default_if: - .. data:: default +- ``apply_default_if`` - The value to be placed instead of a missed one. + A function that specifies whether to apply the default value specified using ``default``. - Example: see :ref:`config_utils_schema_validate_annotation` +.. _config-schema_node_annotation-default: - .. _config-schema_node_annotation-type: +- ``default`` - .. data:: type + The value to be placed instead of a missed one. - A value type. See :ref:`config_utils_schema_data_types`. +.. _config-schema_node_annotation-type: - Example: see :ref:`config_utils_schema_validate_annotation` +- ``type`` - .. _config-schema_node_annotation-validate: + A value type. See :ref:`config_utils_schema_data_types`. - .. data:: validate +.. _config-schema_node_annotation-validate: - A function used to validate data. +- ``validate`` - Example: see :ref:`config_utils_schema_validate_annotation` + A function used to validate data. .. _config-utils-schema_node_object: @@ -793,6 +771,8 @@ schema_node_annotation schema_node_object ~~~~~~~~~~~~~~~~~~ +TODO: add missing fields + .. class:: schema_node_object .. _config-schema_node_object-computed-annotations: From 01cbe38ad27f847a11c97519da5ae5cd5f45cd41 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Thu, 12 Sep 2024 10:28:14 +0300 Subject: [PATCH 11/33] Schema validation - new sample --- .../config.yaml | 13 ++++++++ .../http_api.lua | 32 +++++++++++++++++++ .../instances.yml | 1 + 3 files changed, 46 insertions(+) create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua create mode 100644 doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml new file mode 100644 index 0000000000..d5d003f21f --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml @@ -0,0 +1,13 @@ +groups: + group001: + replicasets: + replicaset001: + instances: + instance001: + roles: [ http_api ] + roles_cfg: + http_api: + listen_address: + host: '127.0.0.1' + port: 8080 + scheme: 'http' diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua new file mode 100644 index 0000000000..12102a3a6b --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua @@ -0,0 +1,32 @@ +-- http_api.lua -- +local log = require('log').new("http_api") +local schema = require('experimental.config.utils.schema') + +local listen_address_schema = schema.new('listen_address', schema.record({ + listen_address = schema.record({ + scheme = schema.enum({ 'http', 'https' }), + host = schema.scalar({ type = 'string' }), + port = schema.scalar({ type = 'integer' }) + }) +})) + +local function validate(cfg) + listen_address_schema:validate(cfg) +end + +local function apply(cfg) + local scheme = listen_address_schema:get(cfg, 'listen_address.scheme') + local host = listen_address_schema:get(cfg, 'listen_address.host') + local port = listen_address_schema:get(cfg, 'listen_address.port') + log.info("HTTP API endpoint: %s://%s:%d", scheme, host, port) +end + +local function stop() + log.info("The 'http_api' role is stopped") +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml new file mode 100644 index 0000000000..aa60c2fc42 --- /dev/null +++ b/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml @@ -0,0 +1 @@ +instance001: From 81da573be96c2a0999ad95b6a83cbbd31de1e829 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Thu, 12 Sep 2024 10:43:34 +0300 Subject: [PATCH 12/33] Schema validation - rename samples --- .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../api.lua | 0 .../config.yaml | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http-api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../config.yaml | 0 .../http_api.lua | 0 .../instances.yml | 0 .../reference_lua/config/utils_schema.rst | 84 +++++++++---------- 28 files changed, 41 insertions(+), 43 deletions(-) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record => config_schema_annotations}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_annotations => config_schema_annotations}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_array_record => config_schema_annotations}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_computed_annotations => config_schema_computed_annotations}/api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_computed_annotations => config_schema_computed_annotations}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record => config_schema_computed_annotations}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_annotations => config_schema_fromenv}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_fromenv => config_schema_fromenv}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_annotations => config_schema_fromenv}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_fromenv => config_schema_methods}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_methods => config_schema_methods}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_computed_annotations => config_schema_methods}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_array_record => config_schema_nodes_array}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_array_record => config_schema_nodes_array}/http-api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_fromenv => config_schema_nodes_array}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_map => config_schema_nodes_map}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_map => config_schema_nodes_map}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_hierarchy => config_schema_nodes_map}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_methods => config_schema_nodes_record}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record => config_schema_nodes_record}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_map => config_schema_nodes_record}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_hierarchy => config_schema_nodes_record_hierarchy}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_hierarchy => config_schema_nodes_record_hierarchy}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_record_methods => config_schema_nodes_record_hierarchy}/instances.yml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_scalar => config_schema_nodes_scalar}/config.yaml (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_scalar => config_schema_nodes_scalar}/http_api.lua (100%) rename doc/code_snippets/snippets/config/instances.enabled/{application_validate_cfg_scalar => config_schema_nodes_scalar}/instances.yml (100%) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_annotations/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_fromenv/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_methods/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/http-api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/http-api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_hierarchy/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/instances.yml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml b/doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/instances.yml similarity index 100% rename from doc/code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/instances.yml rename to doc/code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/instances.yml diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 2a5c396010..e3d788f8cd 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -26,14 +26,14 @@ Getting started with config.utils.schema Example config - scalar type: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml :language: yaml :start-at: roles :dedent: 1. Load the module: - .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua :start-at: config.utils.schema :end-at: config.utils.schema @@ -42,7 +42,7 @@ Example config - scalar type: 2. Define a schema using :ref:`schema.new() `. Example enum: - .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua :start-at: local http_api_schema :end-before: local function validate @@ -51,7 +51,7 @@ Example config - scalar type: 3. Validate config values using ``config.utils.schema.validate()``. For a role, inside the ``validate()`` func: - .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua :start-at: local function validate :end-before: local function apply @@ -59,7 +59,7 @@ Example config - scalar type: 4. Get value. For a role, inside the ``apply()`` func: - .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua :start-at: local function apply :end-before: local function stop @@ -113,14 +113,14 @@ Scalar Example config - scalar type: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml :language: yaml :start-at: roles :dedent: Schema definition: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua :start-at: local http_api_schema :end-before: local function validate @@ -137,21 +137,36 @@ Record Example config: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/config.yaml :language: yaml :start-at: roles :dedent: Schema: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate :dedent: -TODO: add a sample with nested records. It can be used in the ``Processing configuration data`` section. +Example config 2 (nested fields inside ``listen_address``): + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/config.yaml + :language: yaml + :start-at: roles + :dedent: + +Schema: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua + :language: lua + :start-at: local listen_address_schema + :end-before: local function validate + :dedent: + +TODO: This sample can be used in the ``Processing configuration data`` section. @@ -164,14 +179,14 @@ Also: :ref:`schema.set() ` Example config: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/config.yaml :language: yaml :start-at: roles :dedent: Schema: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/http-api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate @@ -184,14 +199,14 @@ Map Example config: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/config.yaml :language: yaml :start-at: roles :dedent: Schema: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_map/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate @@ -223,14 +238,14 @@ Annotations Example config: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/config.yaml :language: yaml :start-at: roles :dedent: Schema definition (``validate``, ``allowed_values``, ``default``): -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate @@ -238,7 +253,7 @@ Schema definition (``validate``, ``allowed_values``, ``default``): Validate functions: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_annotations/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua :start-at: local function validate_host :end-before: local listen_address_schema @@ -254,7 +269,7 @@ User-defined annotations The ``env`` annotation: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function collect_env_cfg @@ -270,7 +285,7 @@ Computed annotations Schema node: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua :start-at: local listen_address :end-before: local http_listen_address_schema @@ -278,7 +293,7 @@ Schema node: Passes validation: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua :start-at: local http_listen_address_schema :end-before: local iproto_listen_address_schema @@ -286,7 +301,7 @@ Passes validation: Raises an error: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_computed_annotations/api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua :start-at: local iproto_listen_address_schema :end-before: local function validate @@ -301,14 +316,14 @@ User-defined methods Example config: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/config.yaml +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_methods/config.yaml :language: yaml :start-at: roles :dedent: Schema: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_methods/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_methods/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate @@ -330,7 +345,7 @@ Validating configuration Role -- inside the ``validate()`` function: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua :start-at: local function validate :end-before: local function apply @@ -342,29 +357,12 @@ Role -- inside the ``validate()`` function: Getting configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Role -- inside the ``apply()`` function (scalar): - -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_scalar/http_api.lua - :language: lua - :start-at: local function apply - :end-before: local function stop - :dedent: - -Record -- pass field names: - -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua :start-at: local function apply :end-before: local function stop :dedent: -Array: - -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_array_record/http-api.lua - :language: lua - :start-at: local function apply - :end-before: local function stop - :dedent: .. _config_utils_schema_transform_configuration: @@ -383,7 +381,7 @@ Parsing environment variables Using :ref:`schema.fromenv() `: -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_validate_cfg_record_fromenv/http_api.lua +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua :start-at: local listen_address_schema :end-before: local function validate From e676fecbc2bc17c9c6ee1175ad886081145e29cd Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Thu, 12 Sep 2024 11:40:31 +0300 Subject: [PATCH 13/33] Schema validation 2 --- doc/reference/reference_lua/config/utils_schema.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index e3d788f8cd..92836b6274 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -372,6 +372,14 @@ Transforming configuration TODO: filter, merge, map, apply_default ? +Example with ``apply_default()``: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua + :language: lua + :start-at: local function apply + :end-before: local function stop + :dedent: + .. _config_utils_schema_env-vars: From f159dddc293a6610797a7e8e0ecdf02adf7df581 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Thu, 12 Sep 2024 13:14:56 +0300 Subject: [PATCH 14/33] Schema validation - add missing fields --- .../reference_lua/config/utils_schema.rst | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 92836b6274..a0066aa676 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -502,15 +502,30 @@ API Reference * - **schema_node_object** - + * - :ref:`schema_node_object.allowed_values ` + - TODO + + * - :ref:`schema_node_object.apply_default_if ` + - TODO + * - :ref:`schema_node_object.computed.annotations ` - TODO + * - :ref:`schema_node_object.default ` + - TODO + * - :ref:`schema_node_object.fields ` - TODO + * - :ref:`schema_node_object.items ` + - TODO + * - :ref:`schema_node_object.type ` - TODO + * - :ref:`schema_node_object.validate ` + - TODO + @@ -777,13 +792,29 @@ schema_node_annotation schema_node_object ~~~~~~~~~~~~~~~~~~ -TODO: add missing fields - .. class:: schema_node_object + .. _config-schema_node_object-allowed_values: + + .. data:: allowed_values + + TODO + + .. _config-schema_node_object-apply_default_if: + + .. data:: apply_default_if + + TODO + .. _config-schema_node_object-computed-annotations: - .. data:: schema_node_object.computed.annotations + .. data:: computed.annotations + + TODO + + .. _config-schema_node_object-default: + + .. data:: default TODO @@ -793,8 +824,20 @@ TODO: add missing fields TODO + .. _config-schema_node_object-items: + + .. data:: items + + TODO + .. _config-schema_node_object-type: .. data:: type TODO + + .. _config-schema_node_object-validate: + + .. data:: validate + + TODO From d18eb5f70b1a01d489a1fdd9e98daf57768eb3c7 Mon Sep 17 00:00:00 2001 From: andreyaksenov Date: Thu, 12 Sep 2024 14:33:15 +0300 Subject: [PATCH 15/33] Schema validation 3 --- .../reference_lua/config/utils_schema.rst | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index a0066aa676..1176392088 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -94,15 +94,14 @@ Scalar and composite types There are scalar and composite types. -- :ref:`schema.scalar() ` -- :ref:`schema.record() ` -- :ref:`schema.map() ` -- :ref:`schema.array() ` +- Scalar type. + Can be created using ``schema.scalar()``. + There is also a shortcut: :ref:`schema.enum() `. + Learn more about supported data types: :ref:`config_utils_schema_data_types`. +- Composite data types: record, array, map. + Can be created using :ref:`schema.record() `, :ref:`schema.array() `, :ref:`schema.map() `. + There is also a shortcut for arrays: :ref:`schema.set() `. -Shortcuts: - -- :ref:`schema.enum() ` -- :ref:`schema.set() ` .. _config_utils_schema_type_system_scalar: @@ -135,7 +134,7 @@ See also: :ref:`config_utils_schema_data_types`. Record ^^^^^^ -Example config: +Example config 1 (no nested fields - only scalars inside the record): .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/config.yaml :language: yaml @@ -218,6 +217,8 @@ Schema: Data types ********** +Passed to ``scalar()``, see :ref:`config_utils_schema_type_system_scalar`. + Supported types: - ``string`` -- ``string`` @@ -236,6 +237,19 @@ Supported types: Annotations *********** +3 groups of annotations: + +- Built-in annotations handled by the module (``validate``, ``allowed_values``, ``default``, ``apply_default_if``). Note that ``validate``, ``allowed_values`` used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. +- User-defined annotations +- Computed annotations + +.. _config_utils_schema_built_in_annotations: + +Built-in annotations +^^^^^^^^^^^^^^^^^^^^ + +TODO: check the ``Built-in annotation`` term. + Example config: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/config.yaml @@ -508,7 +522,7 @@ API Reference * - :ref:`schema_node_object.apply_default_if ` - TODO - * - :ref:`schema_node_object.computed.annotations ` + * - :ref:`schema_node_object.computed ` - TODO * - :ref:`schema_node_object.default ` @@ -745,7 +759,7 @@ schema_object .. data:: schema - TODO + TODO, see also ``schema_node_object`` @@ -806,11 +820,11 @@ schema_node_object TODO - .. _config-schema_node_object-computed-annotations: + .. _config-schema_node_object-computed: - .. data:: computed.annotations + .. data:: computed - TODO + TODO (for example, ``computed.annotations``) .. _config-schema_node_object-default: From 24adbba72ef9a9b1e5f2028ff480766e1b67fcd6 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Wed, 9 Oct 2024 17:33:47 +0700 Subject: [PATCH 16/33] Add text --- .../reference_lua/config/utils_schema.rst | 251 +++++++++++------- 1 file changed, 162 insertions(+), 89 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 1176392088..6e1c965865 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -5,32 +5,35 @@ Submodule experimental.config.utils.schema **Since:** :doc:`3.2.0 ` -Tarantool allows you to provide arbitrary configurations for cluster applications: +The ``experimental.config.utils.schema`` module is used to validate and process +parts of cluster configurations that have arbitrary user-defined structures: -- For applications loaded using the :ref:`app ` option, :ref:`app.cfg ` is used to provide a configuration. -- For :ref:`custom roles ` developed as a part of a cluster application, :ref:`roles_cfg ` is used. +- :ref:`app.cfg ` for applications loaded using the :ref:`app ` option +- :ref:`roles_cfg ` for :ref:`custom roles ` developed as a part of a cluster application -The ``experimental.config.utils.schema`` module can be used to validate such configurations and process their data: get and set configuration values, filter and transform configuration data, and so on. +The module provides an API to get and set configuration values, filter and transform configuration data, and so on. .. important:: ``experimental.config.utils.schema`` is an experimental submodule and is subject to changes. - - .. _config_utils_schema_getting_started: Getting started with config.utils.schema ---------------------------------------- -Example config - scalar type: +As an example, consider an :ref:`application role ` +that has a single configuration option - an HTTP endpoint address. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml :language: yaml :start-at: roles :dedent: +This is how you can use the ``experimental.config.utils.schema`` module to process +the role configuration: + 1. Load the module: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua @@ -39,8 +42,9 @@ Example config - scalar type: :end-at: config.utils.schema :dedent: -2. Define a schema using :ref:`schema.new() `. - Example enum: +2. Define a *schema* -- the root object that stores information about the role's + configuration -- using :ref:`schema.new() `. The example + below shows a schema that includes a single string option: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -48,8 +52,9 @@ Example config - scalar type: :end-before: local function validate :dedent: -3. Validate config values using ``config.utils.schema.validate()``. - For a role, inside the ``validate()`` func: +3. Use the :ref:`schema.validate() ` function to + validate configuration values against the schema. In case of a role, call this + function inside the role's :ref:`validate() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -57,7 +62,8 @@ Example config - scalar type: :end-before: local function apply :dedent: -4. Get value. For a role, inside the ``apply()`` func: +4. Obtain values of configuration options using :ref:`schema.get() `. + In case of a role, call it inside the role's :ref:`apply() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -66,16 +72,17 @@ Example config - scalar type: :dedent: - .. _config_utils_schema_definition: Defining a schema ----------------- -Create using :ref:`schema.new() `. Options: +An application's or a role's *configuration schema* is a core object that stores all +information about a user-defined configuration. To create a schema, use +the :ref:`schema.new() ` function. It has the following options: -- schema name -- schema node: +- schema name -- an arbitrary string +- schema node -- th - scalar and composite (record, array, map) types - annotations (type, validate, and so on) @@ -87,6 +94,7 @@ Create using :ref:`schema.new() `. Options: Schema nodes ~~~~~~~~~~~~ +Schema nodes can have one of two types: *scalar* or *composite*. .. _config_utils_schema_scalar_composite_types: Scalar and composite types @@ -103,21 +111,23 @@ There are scalar and composite types. There is also a shortcut for arrays: :ref:`schema.set() `. -.. _config_utils_schema_type_system_scalar: +.. _config_utils_schema_nodes_scalar: -Scalar -^^^^^^ +Scalar nodes +************ -:ref:`schema.scalar() ` +Scalar nodes hold a single value of a primitive type: a string, a number, a boolean value +and so on. For the full list of available scalar types, see :ref:`config_utils_schema_data_types`. -Example config - scalar type: +This configuration has one scalar node of the ``string`` type: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml :language: yaml :start-at: roles :dedent: -Schema definition: +To define a scalar node in a schema, use :ref:`schema.scalar() `: +The following code defines a configuration schema shown above: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -125,23 +135,27 @@ Schema definition: :end-before: local function validate :dedent: -See also: :ref:`config_utils_schema_data_types`. +Todo: enum? +allowed? +.. _config_utils_schema_nodes_record: +Records +******* -.. _record: +*Record* is a composite node that includes a predefined set of other nodes, scalar +or composite. The names and types of fields in a record are determined by the schema. -Record -^^^^^^ - -Example config 1 (no nested fields - only scalars inside the record): +In YAML, a record is represented as a node with nested fields. +For example, the following configuration has a record node ``http_api`` with +three scalar fields: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/config.yaml :language: yaml :start-at: roles :dedent: -Schema: +The following schema describes this configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua :language: lua @@ -149,15 +163,21 @@ Schema: :end-before: local function validate :dedent: +.. note:: + + Note the use of the :ref:`schema.enum() ` function. + It defines a scalar node with a limited set of allowed values. -Example config 2 (nested fields inside ``listen_address``): +Records are also used to define nested schema nodes of non-primitive types. In the example +below, the ``http_api`` node holds a single composite object -- ``listen_address``. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/config.yaml :language: yaml :start-at: roles :dedent: -Schema: +To create a record node in a schema, use :ref:`schema.record() `. +The following schema describes this configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua @@ -168,22 +188,25 @@ Schema: TODO: This sample can be used in the ``Processing configuration data`` section. - -.. _array_record: +.. _config_utils_schema_nodes_array: Array -^^^^^ +***** -Also: :ref:`schema.set() ` +*Array* is a composite node type that includes a collection of items of the same +type. The type can be either primitive or complex. -Example config: +In YAML, array items start with hyphens. For example, the following configuration +includes an array named ``http_api``. Each its item is a record with three fields: +``host``, ``port``, and ``scheme``: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/config.yaml :language: yaml :start-at: roles :dedent: -Schema: +To create an array node in a schema, use :ref:`schema.array() `. +The following schema describes this configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_array/http-api.lua :language: lua @@ -191,19 +214,28 @@ Schema: :end-before: local function validate :dedent: -.. _record_map: +There is also the :ref:`schema.set() ` function that enables +creating arrays with a limited set of item values. + +.. _config_utils_schema_nodes_map: Map -^^^ +*** + +*Map* is a composite node type that includes key-value pairs with arbitrary values +of predefined types. -Example config: +In YAML, a map is represented as a node with nested fields. +For example, the following configuration has a map node ``endpoints`` with +three items: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/config.yaml :language: yaml :start-at: roles :dedent: -Schema: +To create a map node in a schema, use :ref:`schema.map() `. +The following schema describes this configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/http_api.lua :language: lua @@ -217,27 +249,60 @@ Schema: Data types ********** -Passed to ``scalar()``, see :ref:`config_utils_schema_type_system_scalar`. +Scalar nodes can have the following data types: -Supported types: -- ``string`` -- ``string`` -- ``number`` -- ``number`` -- ``integer`` -- ``number`` -- ``boolean`` -- ``boolean`` -- ``any`` - any accepts an arbitrary Lua type, including ``table``. A scalar of the ``any`` type may be used to declare an arbitrary value that doesn't need any validation. -- ``string, number`` -- ``string`` or ``number`` -- ``number, string`` -- ``string`` or ``string`` +.. container:: table + .. list-table:: + :header-rows: 1 + * - Scalar type + - Lua type + - Comment + * - ``string`` + - ``string`` + - + + * - ``number`` + - ``number`` + - + * - ``integer`` + - ``number`` + - Only integer numbers + + * - ``boolean`` + - ``boolean`` + - ``true`` or ``false`` + + * - ``any`` + - Arbitrary Lua value + - May be used to declare an arbitrary value that doesn't need validation. + + * - |``string, number`` + | or + |``number, string`` + - ``string`` or ``number`` + - .. _config_utils_schema_annotation: Annotations *********** -3 groups of annotations: +Each scalar node is defined by a set of *annotations* -- attributes that set its +parameters: type, default value, validation function, etc + +Annotations are passed as schema.scalar argument rows + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua + :language: lua + :start-at: host + :end-before: port + :dedent: + +Node annotations fall into three groups: - Built-in annotations handled by the module (``validate``, ``allowed_values``, ``default``, ``apply_default_if``). Note that ``validate``, ``allowed_values`` used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. - User-defined annotations @@ -250,14 +315,20 @@ Built-in annotations TODO: check the ``Built-in annotation`` term. -Example config: +Built-in annotations are interpreted by the module itself. There are the following +built-in annotations: + +- + +Consider the following role configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/config.yaml :language: yaml :start-at: roles :dedent: -Schema definition (``validate``, ``allowed_values``, ``default``): +The following schema uses built-in annotations ``default``, ``allowed_values``, and ``validate`` +to define default and allowed option values and validation functions: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua @@ -265,7 +336,7 @@ Schema definition (``validate``, ``allowed_values``, ``default``): :end-before: local function validate :dedent: -Validate functions: +Validation functions can look as follows: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua @@ -274,14 +345,17 @@ Validate functions: :dedent: - - .. _config_utils_schema_user_defined_annotations: User-defined annotations ^^^^^^^^^^^^^^^^^^^^^^^^ -The ``env`` annotation: +A schema node can have *user-defined annotations* with arbitrary names. Such annotations +are used to implement custom behavior. You can get their names and values from +the schema and use in the role or application code. + +In the example below, the user-defined ``env`` annotation is used to provide names +of environment variables that can store values of configuration options: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua @@ -297,7 +371,10 @@ See the full sample here: :ref:`config_utils_schema_env-vars`. Computed annotations ^^^^^^^^^^^^^^^^^^^^ -Schema node: +*Computed annotations* enable access to schema data + +In the example below, the validate function of the listen_address record +uses computed annotation to access the schema data from outside the record: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua @@ -305,6 +382,7 @@ Schema node: :end-before: local http_listen_address_schema :dedent: +The following schema with listen_address passes the validation: Passes validation: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua @@ -313,7 +391,7 @@ Passes validation: :end-before: local iproto_listen_address_schema :dedent: -Raises an error: +If this record is added to a schema with ``protocol = 'iproto'``, an error is raised: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua @@ -328,14 +406,10 @@ Raises an error: User-defined methods ~~~~~~~~~~~~~~~~~~~~ -Example config: +In addition to nodes, a schema can include *methods*. Methods are user-defined +functions that can be called on this schema. -.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_methods/config.yaml - :language: yaml - :start-at: roles - :dedent: - -Schema: +For example, this schema has a method that returns its fields merged in a URI string: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_methods/http_api.lua :language: lua @@ -357,7 +431,11 @@ Processing configuration data Validating configuration ~~~~~~~~~~~~~~~~~~~~~~~~ -Role -- inside the ``validate()`` function: +The schema object's :ref:`validate() ` function performs all the necessary checks +on the provided configuration. It validates the configuration structure, node types, allowed values, +and other aspects defines in the schema. + +When writing roles, call this function inside the :ref:`role validation function `: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua @@ -371,6 +449,9 @@ Role -- inside the ``validate()`` function: Getting configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +To get configuration values, use the schema's :ref:`get() ` method. +The function takes the configuration and the full path to the node as arguments: + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua :start-at: local function apply @@ -401,7 +482,9 @@ Example with ``apply_default()``: Parsing environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Using :ref:`schema.fromenv() `: +The :ref:`schema.fromenv() ` allows getting configuration +values from environment variables. The example below shows how to do this by +adding a user-defined annotation ``env``: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua @@ -409,12 +492,10 @@ Using :ref:`schema.fromenv() `: :end-before: local function validate :dedent: -Other API members: - -- Custom annotation (``env``) -- ``pairs()`` -- ``set()`` +The function also uses schema object methods: +- :ref:`pairs() ` to iterate over the schema nodes. +- :ref:`set() ` to assign configuration values. .. _api-reference-config-utils-schema: @@ -434,28 +515,28 @@ API Reference - * - :ref:`schema.array() ` - - TODO + - Define an array * - :ref:`schema.enum() ` - - TODO + - Create an enum * - :ref:`schema.fromenv() ` - - TODO + - Parse a value from an environment variable * - :ref:`schema.map() ` - - TODO + - Create a map * - :ref:`schema.new() ` - - TODO + - Create a schema * - :ref:`schema.record() ` - - TODO + - Create a record * - :ref:`schema.scalar() ` - - TODO + - Create a scalar * - :ref:`schema.set() ` - - TODO + - Define a set * - **schema_object** - @@ -472,8 +553,6 @@ API Reference * - :ref:`schema_object:map() ` - TODO - - * - :ref:`schema_object:merge() ` - TODO @@ -541,12 +620,6 @@ API Reference - TODO - - - - - - .. _config-utils-schema_functions: Functions From d3b8d90772025b76cf83752ba334dd7c4728de55 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Thu, 10 Oct 2024 16:26:34 +0700 Subject: [PATCH 17/33] Add text --- .../reference_lua/config/utils_schema.rst | 208 +++++++++--------- 1 file changed, 109 insertions(+), 99 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 6e1c965865..869441ca98 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -77,47 +77,44 @@ the role configuration: Defining a schema ----------------- -An application's or a role's *configuration schema* is a core object that stores all -information about a user-defined configuration. To create a schema, use -the :ref:`schema.new() ` function. It has the following options: +A *configuration schema* stores information about a user-defined configuration structure +that can be passed inside an :ref:`app.cfg ` +or a :ref:`roles_cfg ` section. It includes +option names, types, hierarchy, and other aspects of a configuration. -- schema name -- an arbitrary string -- schema node -- th +To create a schema, use the :ref:`schema.new() ` function. +It has the following arguments: - - scalar and composite (record, array, map) types - - annotations (type, validate, and so on) - -- (optional) methods +- schema name -- an arbitrary string to use as an identifier +- schema nodes -- a hierarchical structure of configuration options that form + the schema. +- (optional) methods -- functions that can be called on this schema object. .. _config_utils_schema_nodes: Schema nodes ~~~~~~~~~~~~ -Schema nodes can have one of two types: *scalar* or *composite*. -.. _config_utils_schema_scalar_composite_types: - -Scalar and composite types -************************** +Schema nodes describe the hierarchy of options within a schema. There are two types of schema nodes: -There are scalar and composite types. +- *Scalar* nodes hold a single value of a supported primitive type. For example, + a string configuration option of a role is a scalar node its schema. +- *Composite* nodes include multiple values in different forms: records, arrays, or maps. -- Scalar type. - Can be created using ``schema.scalar()``. - There is also a shortcut: :ref:`schema.enum() `. - Learn more about supported data types: :ref:`config_utils_schema_data_types`. -- Composite data types: record, array, map. Can be created using :ref:`schema.record() `, :ref:`schema.array() `, :ref:`schema.map() `. There is also a shortcut for arrays: :ref:`schema.set() `. +Each schema node is defined by its *annotations* + - annotations (type, validate, and so on) + .. _config_utils_schema_nodes_scalar: Scalar nodes ************ -Scalar nodes hold a single value of a primitive type: a string, a number, a boolean value -and so on. For the full list of available scalar types, see :ref:`config_utils_schema_data_types`. +Scalar nodes hold a single value of a primitive type, for example, a string or a number. +For the full list of supported scalar types, see :ref:`config_utils_schema_data_types`. This configuration has one scalar node of the ``string`` type: @@ -135,8 +132,61 @@ The following code defines a configuration schema shown above: :end-before: local function validate :dedent: -Todo: enum? -allowed? +If a scalar node has a limited set of allowed values, you can also define it with +the :ref:`schema.enum() `. Pass the list of allowed values as +its argument: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua + :language: lua + :start-at: scheme + :end-before: host + :dedent: + +.. note:: + + Another way to restrict possible option values is the :ref:`allowed_values ` + built-in annotation. + +.. _config_utils_schema_data_types: + +Data types +^^^^^^^^^^ + +Scalar nodes can have the following data types: + +.. container:: table + + .. list-table:: + :header-rows: 1 + + * - Scalar type + - Lua type + - Comment + + * - ``string`` + - ``string`` + - + + * - ``number`` + - ``number`` + - + * - ``integer`` + - ``number`` + - Only integer numbers + + * - ``boolean`` + - ``boolean`` + - ``true`` or ``false`` + + * - ``string, number`` + or + ``number, string`` + - ``string`` or ``number`` + - + + * - ``any`` + - Arbitrary Lua value + - May be used to declare an arbitrary value that doesn't need validation. .. _config_utils_schema_nodes_record: @@ -144,9 +194,7 @@ Records ******* *Record* is a composite node that includes a predefined set of other nodes, scalar -or composite. The names and types of fields in a record are determined by the schema. - -In YAML, a record is represented as a node with nested fields. +or composite. In YAML, a record is represented as a node with nested fields. For example, the following configuration has a record node ``http_api`` with three scalar fields: @@ -155,7 +203,8 @@ three scalar fields: :start-at: roles :dedent: -The following schema describes this configuration: +To define a record node in a schema, use :ref:`schema.record() `: +The following schema describes the configuration above: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua :language: lua @@ -163,11 +212,6 @@ The following schema describes this configuration: :end-before: local function validate :dedent: -.. note:: - - Note the use of the :ref:`schema.enum() ` function. - It defines a scalar node with a limited set of allowed values. - Records are also used to define nested schema nodes of non-primitive types. In the example below, the ``http_api`` node holds a single composite object -- ``listen_address``. @@ -176,7 +220,6 @@ below, the ``http_api`` node holds a single composite object -- ``listen_address :start-at: roles :dedent: -To create a record node in a schema, use :ref:`schema.record() `. The following schema describes this configuration: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua @@ -194,7 +237,7 @@ Array ***** *Array* is a composite node type that includes a collection of items of the same -type. The type can be either primitive or complex. +type. The items can be either scalar or composite nodes. In YAML, array items start with hyphens. For example, the following configuration includes an array named ``http_api``. Each its item is a record with three fields: @@ -215,7 +258,7 @@ The following schema describes this configuration: :dedent: There is also the :ref:`schema.set() ` function that enables -creating arrays with a limited set of item values. +creating arrays with a limited set of allowed items. .. _config_utils_schema_nodes_map: @@ -226,8 +269,7 @@ Map of predefined types. In YAML, a map is represented as a node with nested fields. -For example, the following configuration has a map node ``endpoints`` with -three items: +For example, the following configuration has the ``endpoints`` node: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/config.yaml :language: yaml @@ -235,7 +277,8 @@ three items: :dedent: To create a map node in a schema, use :ref:`schema.map() `. -The following schema describes this configuration: +If this node is declared as a map as shown below, the ``endpoints`` section can include +any number of options with arbitrary names and boolean values. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_map/http_api.lua :language: lua @@ -243,58 +286,16 @@ The following schema describes this configuration: :end-before: local function validate :dedent: - -.. _config_utils_schema_data_types: - -Data types -********** - -Scalar nodes can have the following data types: - - -.. container:: table - - .. list-table:: - :header-rows: 1 - - * - Scalar type - - Lua type - - Comment - - * - ``string`` - - ``string`` - - - - * - ``number`` - - ``number`` - - - * - ``integer`` - - ``number`` - - Only integer numbers - - * - ``boolean`` - - ``boolean`` - - ``true`` or ``false`` - - * - ``any`` - - Arbitrary Lua value - - May be used to declare an arbitrary value that doesn't need validation. - - * - |``string, number`` - | or - |``number, string`` - - ``string`` or ``number`` - - - .. _config_utils_schema_annotation: Annotations *********** -Each scalar node is defined by a set of *annotations* -- attributes that set its -parameters: type, default value, validation function, etc +Node *annotations* are attributes that define the its various aspects. For example, +scalar nodes have a required annotation ``type`` that defines the option type. +Other annotations can optionally set default values, validation function, and -Annotations are passed as schema.scalar argument rows +Annotations are passed in a table to the node creation function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua @@ -304,9 +305,12 @@ Annotations are passed as schema.scalar argument rows Node annotations fall into three groups: -- Built-in annotations handled by the module (``validate``, ``allowed_values``, ``default``, ``apply_default_if``). Note that ``validate``, ``allowed_values`` used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. -- User-defined annotations -- Computed annotations +- *Built-in annotations* are handled by the module. These are: ``type``, ``validate``, ``allowed_values``, ``default`` and ``apply_default_if``. + Note that ``validate``, ``allowed_values`` are used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. +- *User-defined annotations* add named node attributes that can be used in the + application or role code. +- *Computed annotations* allow access to annotations of other nodes throughout + the schema. .. _config_utils_schema_built_in_annotations: @@ -318,7 +322,11 @@ TODO: check the ``Built-in annotation`` term. Built-in annotations are interpreted by the module itself. There are the following built-in annotations: -- +- ``type`` -- the node value type. Mandatory for scalar nodes, except those created with ``schema.enum``. +- ``allowed_values`` -- a list of possible node values. +- ``validate`` -- a validation function for the provided node value. +- ``default`` -- a value to use if the option is not specified in the configuration. +- ``apply_default_if`` -- a function that defines when to apply the default value. Consider the following role configuration: @@ -371,10 +379,10 @@ See the full sample here: :ref:`config_utils_schema_env-vars`. Computed annotations ^^^^^^^^^^^^^^^^^^^^ -*Computed annotations* enable access to schema data +*Computed annotations* enable access from a node to annotations of its ancestor nodes. -In the example below, the validate function of the listen_address record -uses computed annotation to access the schema data from outside the record: +In the example below, the ``listen_address`` record validation function refers to the +``protocol`` annotation of its ancestor node: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua @@ -382,8 +390,12 @@ uses computed annotation to access the schema data from outside the record: :end-before: local http_listen_address_schema :dedent: -The following schema with listen_address passes the validation: -Passes validation: +.. note:: + + If there are several ancestor nodes with this annotation, its value is taken + from the closest one to the current node. + +The following schema with ``listen_address`` passes the validation: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua @@ -391,7 +403,8 @@ Passes validation: :end-before: local iproto_listen_address_schema :dedent: -If this record is added to a schema with ``protocol = 'iproto'``, an error is raised: +If this record is added to a schema with ``protocol = 'iproto'``, the ``listen_address`` +validation fails with an error: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_computed_annotations/api.lua :language: lua @@ -406,8 +419,7 @@ If this record is added to a schema with ``protocol = 'iproto'``, an error is ra User-defined methods ~~~~~~~~~~~~~~~~~~~~ -In addition to nodes, a schema can include *methods*. Methods are user-defined -functions that can be called on this schema. +A schema can implement custom logic with *methods* -- user-defined functions that can be called on this schema. For example, this schema has a method that returns its fields merged in a URI string: @@ -419,8 +431,6 @@ For example, this schema has a method that returns its fields merged in a URI st - - .. _config_utils_schema_process_data: Processing configuration data From e45f78e38b842b86ac0e85b310c87c4e45adcfc4 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Mon, 14 Oct 2024 17:14:14 +0700 Subject: [PATCH 18/33] Add text --- .../reference_lua/config/utils_schema.rst | 213 +++++++++++++----- 1 file changed, 159 insertions(+), 54 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 869441ca98..7961064bf3 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -233,8 +233,8 @@ TODO: This sample can be used in the ``Processing configuration data`` section. .. _config_utils_schema_nodes_array: -Array -***** +Arrays +****** *Array* is a composite node type that includes a collection of items of the same type. The items can be either scalar or composite nodes. @@ -262,8 +262,8 @@ creating arrays with a limited set of allowed items. .. _config_utils_schema_nodes_map: -Map -*** +Maps +**** *Map* is a composite node type that includes key-value pairs with arbitrary values of predefined types. @@ -525,7 +525,7 @@ API Reference - * - :ref:`schema.array() ` - - Define an array + - Create an array * - :ref:`schema.enum() ` - Create an enum @@ -588,25 +588,25 @@ API Reference - * - :ref:`allowed_values ` - - TODO + - Allowed node values * - :ref:`apply_default_if ` - TODO * - :ref:`default ` - - TODO + - Default node value * - :ref:`type ` - - TODO + - Value type * - :ref:`validate ` - - TODO + - Validation function * - **schema_node_object** - * - :ref:`schema_node_object.allowed_values ` - - TODO + - Allowed node values * - :ref:`schema_node_object.apply_default_if ` - TODO @@ -641,7 +641,7 @@ Functions .. function:: schema.array(opts) - Define an array. + Create an array node of a configuration schema. :param table opts: a table in the following format: @@ -651,31 +651,42 @@ Functions See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + :return: the created schema node + :rtype: table + + See also: :ref:`config_utils_schema_nodes_array` + .. _config-utils-schema-enum: .. function:: schema.enum(allowed_values, annotations) - Create an enum. + A shortcut for creating a string scalar node with a limited set of allowed values. - :param table allowed_values: allowed values + :param table allowed_values: a list of enum members -- values allowed for the node :param table annotations: annotations (see :ref:`schema_node_annotation `) + :return: the created schema node + :rtype: table + + See also: :ref:`config_utils_schema_nodes_scalar` + .. _config-utils-schema-fromenv: .. function:: schema.fromenv(env_var_name, raw_value, schema_node) Parse data from an environment variable as a value of the given type. - :param string env_var_name: env var name - :param string raw_value: raw value + :param string env_var_name: environment variable name + :param string raw_value: environment variable value :param schema_node_object schema_node: a schema node (see :ref:`schema_node_object `) + See also :ref:`config_utils_schema_env-vars` .. _config-utils-schema-map: .. function:: schema.map(opts) - Create a map. + Create a map node of a configuration schema. :param table opts: a table in the following format: @@ -685,9 +696,11 @@ Functions See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. - :return: a table that represents the created schema node + :return: the created schema node :rtype: table + See also: :ref:`config_utils_schema_nodes_map` + .. _config-utils-schema-new: .. function:: schema.new(schema_name, schema_node[, { methods = <...> }]) @@ -695,18 +708,19 @@ Functions Create a schema object. :param string schema_name: a name - :param table schema_node: a node + :param table schema_node: a root schema node :param table methods: methods :return: a new schema instance (see :ref:`schema_object `) :rtype: userdata + See also: :ref:`config_utils_schema_getting_started` .. _config-utils-schema-record: .. function:: schema.record(fields[, annotations]) - Create a record. + Create a record node of a configuration schema. :param table fields: a table of fields in the following format: @@ -718,34 +732,38 @@ Functions :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: a table that represents the created schema node + :return: the created schema node :rtype: table + See also: :ref:`config_utils_schema_nodes_record` .. _config-utils-schema-scalar: .. function:: schema.scalar(type[, annotations]) - Create a scalar. + Create a scalar node of a configuration schema. :param string type: data type (see :ref:`config_utils_schema_data_types`) :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: a table that represents the created schema node + :return: the created schema node :rtype: table + See also: :ref:`config_utils_schema_nodes_scalar` + .. _config-utils-schema-set: .. function:: schema.set(allowed_values, annotations) - Shortcut for array of unique string values from the given list of allowed values. + Shortcut for creating an array node of unique string values from the given list of allowed values. - :param table allowed_values: allowed values + :param table allowed_values: allowed values of array items :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: a table that represents the created schema node + :return: the created schema node :rtype: table + See also: :ref:`config_utils_schema_nodes_array` .. _config-utils-schema_object: @@ -758,17 +776,52 @@ schema_object .. method:: apply_default(data) - Apply default values. + Apply default values. The functions takes the ``default`` + built-in annotation values of the scalar nodes and applies them based + on the ``apply_default_if`` annotation. If there is no ``apply_default_if`` + annotation on a node, the default value is also applied. - :param any data: data + .. note:: - :return: new data + The method works for static defaults. To define a dynamic default value, + use the :ref:`map() ` method. + + :param any data: configuration data + + :return: configuration data with applied schema defaults .. _config-schema_object-filter: .. method:: filter(data, f) - Filter data based on the schema annotations. + Filter data based on the schema annotations. The methods returns an iterator + by configuration nodes for which the given filter function ``f`` returns ``true``. + + The filter function f receives the following table as the argument: + + .. code-block:: lua + + w = { + path = , + schema = , + data = , + } + + The filter function returns a boolean value that is interpreted as 'accepted' or 'not accepted'. + + Example: call a function on all schema nodes that have the ``my_annotation`` + annotation defined: + + .. code-block:: lua + + s:filter(function(w) + return w.schema.my_annotation ~= nil + end):each(function(w) + do_something(w.data) + end) + + :param any data: configuration data + :param function f: filter function :return: a luafun iterator @@ -776,10 +829,16 @@ schema_object .. method:: get(data, path) - Get nested data that is pointed by the given path. + Get nested data that is pointed by the given path. The path can be + either a dot-separated string (``http.scheme``) or an array-like table (``{ 'http', 'scheme'}``). Example: see :ref:`config_utils_schema_get_configuration` + :param any data: configuration data + :param any path: path to the target node + + :return: data at the given path + .. _config-schema_object-map: .. method:: map(data, f, f_ctx) @@ -796,10 +855,10 @@ schema_object Merge two hierarchical values (prefer the latter). - :param any a: data - :param any b: data + :param any a: configuration data + :param any b: configuration data - :return: new data + :return: merged configuration data .. _config-schema_object-pairs: @@ -811,41 +870,55 @@ schema_object .. _config-schema_object-set: - .. method:: set() + .. method:: set(data, path, value) - TODO + Set a given value at the given path in a configuration. + The path can be either a dot-separated string (``http.scheme``) or + an array-like table (``{ 'http', 'scheme'}``). - .. _config-schema_object-methods: + :param any data: configuration data + :param any path: path to the target node + :param any value: new value + + :return: updated configuration data .. _config-schema_object-validate: .. method:: validate(data) - Validate data against the schema. - ``validate()`` raises an error if the specified data doesn't adhere this schema. + Validate data against the schema. If the data doesn't adhere to the schema, + an error is raised. + + The method checks the types of ``data`` fields against the schema node types. + Additionally, it checks the data using the ``allowed_values`` and ``validate`` + built-in annotations. :param any data: data Example: see :ref:`config_utils_schema_annotation` + Example: see :ref:`config_utils_schema_validating_configuration` + + .. _config-schema_object-methods: .. data:: methods - TODO + User-defines methods in the schema. + + See also: :ref:`config_utils_schema_methods` .. _config-schema_object-name: .. data:: name - TODO + Schema name. .. _config-schema_object-schema: .. data:: schema - TODO, see also ``schema_node_object`` - - + Schema nodes hierarchy. + See also ``schema_node_object`` .. _config-utils-schema_node_annotation: @@ -857,7 +930,7 @@ schema_node_annotation - ``allowed_values`` - A list of allowed values. + A list of allowed values. See also :ref:`config-schema_object-validate` .. _config-schema_node_annotation-apply_default_if: @@ -865,24 +938,54 @@ schema_node_annotation A function that specifies whether to apply the default value specified using ``default``. + See also: :ref:`config-schema_object-apply_default` + .. _config-schema_node_annotation-default: - ``default`` - The value to be placed instead of a missed one. + A default value to use for a scalar node if it's not specified explicitly. + Example: :ref:`config_utils_schema_transform_configuration` + + See also: :ref:`config-schema_object-apply_default` .. _config-schema_node_annotation-type: - ``type`` - A value type. See :ref:`config_utils_schema_data_types`. + A value type. See also :ref:`config_utils_schema_data_types`. .. _config-schema_node_annotation-validate: - ``validate`` - A function used to validate data. + A function used to validate node data. + + The function should accept the following arguments: + + See also :ref:`config-schema_object-validate` + + Example: + + .. code-block:: lua + + local schema = require('experimental.config.utils.schema') + + local function validate_email(email, w) + if email:find('@') == nil then + w.error('A email must contain @ symbol, got %q', email) + end + end + + local personal_info_schema = schema.new('personal_info', schema.record({ + email = schema.scalar({ + type = 'string', + validate = validate_email, + }), + })) + personal_info_schema:validate({email = 'foo'}) + -- error: [personal_info] email: A email must contain @ symbol, got "foo" .. _config-utils-schema_node_object: @@ -895,13 +998,13 @@ schema_node_object .. data:: allowed_values - TODO + A list of values allowed for the node. .. _config-schema_node_object-apply_default_if: .. data:: apply_default_if - TODO + A .. _config-schema_node_object-computed: @@ -913,28 +1016,30 @@ schema_node_object .. data:: default - TODO + The default value. .. _config-schema_node_object-fields: .. data:: fields - TODO + Child nodes for record nodes. + See also :ref:`config_utils_schema_nodes_record`. .. _config-schema_node_object-items: .. data:: items - TODO + Node items for array nodes. + See also :ref:`config_utils_schema_nodes_array` .. _config-schema_node_object-type: .. data:: type - TODO + Scalar node type. See :ref:`config_utils_schema_data_types` .. _config-schema_node_object-validate: .. data:: validate - TODO + Node value validation function. From eefb3a4e674c065bd92062cc5df6e0d192b10ba9 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Mon, 14 Oct 2024 18:38:09 +0700 Subject: [PATCH 19/33] Add text --- .../reference_lua/config/utils_schema.rst | 105 ++++++++---------- 1 file changed, 49 insertions(+), 56 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 7961064bf3..6b70c53a14 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -52,7 +52,7 @@ the role configuration: :end-before: local function validate :dedent: -3. Use the :ref:`schema.validate() ` function to +3. Use the :ref:`schema.validate() ` function to validate configuration values against the schema. In case of a role, call this function inside the role's :ref:`validate() ` function: @@ -62,7 +62,7 @@ the role configuration: :end-before: local function apply :dedent: -4. Obtain values of configuration options using :ref:`schema.get() `. +4. Obtain values of configuration options using :ref:`schema.get() `. In case of a role, call it inside the role's :ref:`apply() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua @@ -86,8 +86,8 @@ To create a schema, use the :ref:`schema.new() ` functi It has the following arguments: - schema name -- an arbitrary string to use as an identifier -- schema nodes -- a hierarchical structure of configuration options that form - the schema. +- root schema node -- a table describing the hierarchical schema structure + starting from the root. - (optional) methods -- functions that can be called on this schema object. .. _config_utils_schema_nodes: @@ -99,14 +99,10 @@ Schema nodes describe the hierarchy of options within a schema. There are two ty - *Scalar* nodes hold a single value of a supported primitive type. For example, a string configuration option of a role is a scalar node its schema. -- *Composite* nodes include multiple values in different forms: records, arrays, or maps. - - Can be created using :ref:`schema.record() `, :ref:`schema.array() `, :ref:`schema.map() `. - There is also a shortcut for arrays: :ref:`schema.set() `. - -Each schema node is defined by its *annotations* - - annotations (type, validate, and so on) +- *Composite* nodes include multiple values in different forms: *records*, *arrays*, or *maps*. +A node can have *annotations* -- named attributes that enable customization of +its behavior, for example, setting a default value. .. _config_utils_schema_nodes_scalar: @@ -123,8 +119,8 @@ This configuration has one scalar node of the ``string`` type: :start-at: roles :dedent: -To define a scalar node in a schema, use :ref:`schema.scalar() `: -The following code defines a configuration schema shown above: +To define a scalar node in a schema, use :ref:`schema.scalar() `. +The following schema can be used to process the configuration shown above: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -213,7 +209,7 @@ The following schema describes the configuration above: :dedent: Records are also used to define nested schema nodes of non-primitive types. In the example -below, the ``http_api`` node holds a single composite object -- ``listen_address``. +below, the ``http_api`` node includes another record ``listen_address``. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/config.yaml :language: yaml @@ -228,8 +224,6 @@ The following schema describes this configuration: :end-before: local function validate :dedent: -TODO: This sample can be used in the ``Processing configuration data`` section. - .. _config_utils_schema_nodes_array: @@ -265,7 +259,7 @@ creating arrays with a limited set of allowed items. Maps **** -*Map* is a composite node type that includes key-value pairs with arbitrary values +*Map* is a composite node type that holds an arbitrary number of key-value pairs of predefined types. In YAML, a map is represented as a node with nested fields. @@ -291,16 +285,16 @@ any number of options with arbitrary names and boolean values. Annotations *********** -Node *annotations* are attributes that define the its various aspects. For example, -scalar nodes have a required annotation ``type`` that defines the option type. +Node *annotations* are named attributes that define the its various aspects. For example, +scalar nodes have a required annotation ``type`` that defines the node value type. Other annotations can optionally set default values, validation function, and Annotations are passed in a table to the node creation function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua - :start-at: host - :end-before: port + :start-at: scheme = schema.scalar + :end-before: host = schema.scalar :dedent: Node annotations fall into three groups: @@ -317,16 +311,14 @@ Node annotations fall into three groups: Built-in annotations ^^^^^^^^^^^^^^^^^^^^ -TODO: check the ``Built-in annotation`` term. - Built-in annotations are interpreted by the module itself. There are the following built-in annotations: -- ``type`` -- the node value type. Mandatory for scalar nodes, except those created with ``schema.enum``. -- ``allowed_values`` -- a list of possible node values. -- ``validate`` -- a validation function for the provided node value. -- ``default`` -- a value to use if the option is not specified in the configuration. -- ``apply_default_if`` -- a function that defines when to apply the default value. +- :ref:`type ` -- the node value type. Mandatory for scalar nodes, except for those created with ``schema.enum()``. +- :ref:`allowed_values ` -- a list of possible node values. +- :ref:`validate ` -- a validation function for the provided node value. +- :ref:`default ` -- a value to use if the option is not specified in the configuration. +- :ref:`apply_default_if ` -- a function that defines when to apply the default value. Consider the following role configuration: @@ -362,8 +354,8 @@ A schema node can have *user-defined annotations* with arbitrary names. Such ann are used to implement custom behavior. You can get their names and values from the schema and use in the role or application code. -In the example below, the user-defined ``env`` annotation is used to provide names -of environment variables that can store values of configuration options: +Example: the ``env`` user-defined annotation is used to provide names +of environment variables from which the configuration values can be taken. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua @@ -469,15 +461,17 @@ The function takes the configuration and the full path to the node as arguments: :dedent: - .. _config_utils_schema_transform_configuration: Transforming configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~ -TODO: filter, merge, map, apply_default ? +The module provides methods that transform configuration data based on the schema, +for example, :ref:`apply_default() `, +:ref:`merge() `, ref:`set() `. -Example with ``apply_default()``: +The following sample shows how to apply default values from the schema to fill +missing configuration fields: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_annotations/http_api.lua :language: lua @@ -486,7 +480,6 @@ Example with ``apply_default()``: :dedent: - .. _config_utils_schema_env-vars: Parsing environment variables @@ -546,43 +539,43 @@ API Reference - Create a scalar * - :ref:`schema.set() ` - - Define a set + - Create a set * - **schema_object** - * - :ref:`schema_object:apply_default() ` - - TODO + - Apply default values * - :ref:`schema_object:filter() ` - - TODO + - Filter schema nodes * - :ref:`schema_object:get() ` - - TODO + - Get specified configuration data * - :ref:`schema_object:map() ` - TODO * - :ref:`schema_object:merge() ` - - TODO + - Merge two configurations * - :ref:`schema_object:pairs() ` - - TODO + - Traverse a configuration * - :ref:`schema_object:set() ` - - TODO + - Set a configuration value * - :ref:`schema_object:validate() ` - - TODO + - Validate a configuration against a schema * - :ref:`schema_object.methods ` - - TODO + - User-defined methods * - :ref:`schema_object.name ` - - TODO + - Schema name * - :ref:`schema_object.schema ` - - TODO + - Schema nodes hierarchy * - **schema_node_annotation** - @@ -591,7 +584,7 @@ API Reference - Allowed node values * - :ref:`apply_default_if ` - - TODO + - Condition to apply defaults * - :ref:`default ` - Default node value @@ -609,25 +602,25 @@ API Reference - Allowed node values * - :ref:`schema_node_object.apply_default_if ` - - TODO + - Condition to apply defaults * - :ref:`schema_node_object.computed ` - - TODO + - Computed annotations * - :ref:`schema_node_object.default ` - - TODO + - Default value * - :ref:`schema_node_object.fields ` - - TODO + - Record node fields * - :ref:`schema_node_object.items ` - - TODO + - Array node items * - :ref:`schema_node_object.type ` - - TODO + - Scalar node type * - :ref:`schema_node_object.validate ` - - TODO + - Validation function .. _config-utils-schema_functions: @@ -1010,13 +1003,13 @@ schema_node_object .. data:: computed - TODO (for example, ``computed.annotations``) + ``computed.annotations`` stores the node's :ref:`computed annotations `. .. _config-schema_node_object-default: .. data:: default - The default value. + Node's default value. .. _config-schema_node_object-fields: @@ -1036,7 +1029,7 @@ schema_node_object .. data:: type - Scalar node type. See :ref:`config_utils_schema_data_types` + Node type for scalar nodes. See :ref:`config_utils_schema_data_types` .. _config-schema_node_object-validate: From b573e46ab6aacea0a04e93609470dc342b2d2d39 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Tue, 15 Oct 2024 18:06:07 +0700 Subject: [PATCH 20/33] Add text --- .../reference_lua/config/utils_schema.rst | 183 ++++++++++++++---- 1 file changed, 142 insertions(+), 41 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 6b70c53a14..3d39559d04 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -554,7 +554,7 @@ API Reference - Get specified configuration data * - :ref:`schema_object:map() ` - - TODO + - Transform configuration data * - :ref:`schema_object:merge() ` - Merge two configurations @@ -647,7 +647,7 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_array` + **See also:** :ref:`config_utils_schema_nodes_array` .. _config-utils-schema-enum: @@ -661,19 +661,38 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_scalar` + **See also:** :ref:`config_utils_schema_nodes_scalar` .. _config-utils-schema-fromenv: .. function:: schema.fromenv(env_var_name, raw_value, schema_node) - Parse data from an environment variable as a value of the given type. + Parse an environment variable as a value of the given schema node. + The ``env_var_name`` parameter is used only for error messages. + The value (``raw_value``) should be received using ``os.getenv()`` or ``os.environ()``. + + How the raw value is parsed depends on the ``schema_node`` type: + + - Scalar: + - ``string``: return the value as is + - ``number`` or ``integer: parse the value as a number or an integer + - ``string, number``: attempt to parse as a number; in case of a failure + return the value as is + - ``boolean``: accept ``true`` or ``1`` for ``true``, ``false`` or ``0`` for ``false`` + - ``any``: parse the value as a JSON + - Map: parse either as JSON (if the raw value starts with ``{``) + or as a comma-separated string of ``key=value`` pairs: ``key1=value1,key2=value2`` + - Array: parse either as JSON (if the raw value starts with ``[``) + or as a comma-separated string of items: ``item1,item2,item3`` :param string env_var_name: environment variable name :param string raw_value: environment variable value :param schema_node_object schema_node: a schema node (see :ref:`schema_node_object `) - See also :ref:`config_utils_schema_env-vars` + :return: the created schema node + :rtype: table + + **See also:** :ref:`config_utils_schema_env-vars` .. _config-utils-schema-map: @@ -692,7 +711,7 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_map` + **See also:** :ref:`config_utils_schema_nodes_map` .. _config-utils-schema-new: @@ -707,7 +726,7 @@ Functions :return: a new schema instance (see :ref:`schema_object `) :rtype: userdata - See also: :ref:`config_utils_schema_getting_started` + **See also:** :ref:`config_utils_schema_getting_started` .. _config-utils-schema-record: @@ -728,7 +747,7 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_record` + **See also:** :ref:`config_utils_schema_nodes_record` .. _config-utils-schema-scalar: @@ -742,7 +761,7 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_scalar` + **See also:** :ref:`config_utils_schema_nodes_scalar` .. _config-utils-schema-set: @@ -756,7 +775,7 @@ Functions :return: the created schema node :rtype: table - See also: :ref:`config_utils_schema_nodes_array` + **See also:** :ref:`config_utils_schema_nodes_array` .. _config-utils-schema_object: @@ -769,7 +788,7 @@ schema_object .. method:: apply_default(data) - Apply default values. The functions takes the ``default`` + Apply default values to scalar nodes. The functions takes the ``default`` built-in annotation values of the scalar nodes and applies them based on the ``apply_default_if`` annotation. If there is no ``apply_default_if`` annotation on a node, the default value is also applied. @@ -783,14 +802,16 @@ schema_object :return: configuration data with applied schema defaults - .. _config-schema_object-filter: + **See also:** :ref:`config-schema_node_annotation-default`, :ref:`config-schema_node_annotation-apply_default_if` + +.. _config-schema_object-filter: .. method:: filter(data, f) Filter data based on the schema annotations. The methods returns an iterator by configuration nodes for which the given filter function ``f`` returns ``true``. - The filter function f receives the following table as the argument: + The filter function ``f`` receives the following table as the argument: .. code-block:: lua @@ -800,9 +821,11 @@ schema_object data = , } - The filter function returns a boolean value that is interpreted as 'accepted' or 'not accepted'. + The filter function returns a boolean value that is interpreted as "accepted" or "not accepted". - Example: call a function on all schema nodes that have the ``my_annotation`` + **Example:** + + Calling a function on all schema nodes that have the ``my_annotation`` annotation defined: .. code-block:: lua @@ -822,16 +845,24 @@ schema_object .. method:: get(data, path) - Get nested data that is pointed by the given path. The path can be + Get nested configuration values at the given path. The path can be either a dot-separated string (``http.scheme``) or an array-like table (``{ 'http', 'scheme'}``). - Example: see :ref:`config_utils_schema_get_configuration` + **Example:** + + .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua + :language: lua + :start-at: local scheme + :end-before: local host + :dedent: :param any data: configuration data :param any path: path to the target node :return: data at the given path + **See also**: see :ref:`config_utils_schema_get_configuration` + .. _config-schema_object-map: .. method:: map(data, f, f_ctx) @@ -844,9 +875,24 @@ schema_object .. _config-schema_object-merge: - .. method:: merge(a, b) + .. method:: merge(data_a, data_b) + + Merge two configurations. The method merges configurations in a single + node hierarchy, preferring the latter in case of a collision. + + The following merge rules are used: + + - any present value is preferred over ``nil`` and ``box.NULL`` + - ``box.NULL`` is preferred over ``nil`` + - for scalar and array nodes, the right-hand value is used - Merge two hierarchical values (prefer the latter). + .. important:: + + Note that the method doesn't concatenate arrays. Left hand array + items are discarded. + + - records and maps are deeply merged, that is, the merge is performed + recursively for their nested nodes :param any a: configuration data :param any b: configuration data @@ -857,10 +903,24 @@ schema_object .. method:: pairs() - Traverse the schema. + Walk over the schema and return scalar, array and map schema nodes + + .. important:: + + The method doesn't return records nodes. :return: a luafun iterator + **Example:** + + .. code-block:: lua + + for _, w in schema:pairs() do + local path = w.path + local schema = w.schema + -- <...> + end + .. _config-schema_object-set: .. method:: set(data, path, value) @@ -875,6 +935,8 @@ schema_object :return: updated configuration data + **Example:** see :ref:`config_utils_schema_env-vars` + .. _config-schema_object-validate: .. method:: validate(data) @@ -882,22 +944,28 @@ schema_object Validate data against the schema. If the data doesn't adhere to the schema, an error is raised. - The method checks the types of ``data`` fields against the schema node types. - Additionally, it checks the data using the ``allowed_values`` and ``validate`` - built-in annotations. + The method performs the following checks: + + - field type checks: field values are checked against the schema node types + - allowed values: if a node has the :ref:`config-schema_node_annotation-allowed_values` + annotations of schema nodes, the corresponding data field is checked + against the allowed values list + - validation functions: if a validation function is defined for a node + (the :ref:`config-schema_node_annotation-validate` annotation), it is + executed to check that the provided value is valid. :param any data: data - Example: see :ref:`config_utils_schema_annotation` - Example: see :ref:`config_utils_schema_validating_configuration` + **Example:** see :ref:`config_utils_schema_annotation` and + :ref:`config_utils_schema_validating_configuration` .. _config-schema_object-methods: .. data:: methods - User-defines methods in the schema. + User-defined methods in the schema. - See also: :ref:`config_utils_schema_methods` + **See also:** :ref:`config_utils_schema_methods` .. _config-schema_object-name: @@ -911,7 +979,7 @@ schema_object Schema nodes hierarchy. - See also ``schema_node_object`` + **See also:** ``schema_node_object`` .. _config-utils-schema_node_annotation: @@ -919,46 +987,75 @@ schema_object schema_node_annotation ~~~~~~~~~~~~~~~~~~~~~~ +The following elements of tables passed as node constructor arguments are +parsed by the modules as :ref:`built-in annotations `: + .. _config-schema_node_annotation-allowed_values: - ``allowed_values`` - A list of allowed values. See also :ref:`config-schema_object-validate` + A list of allowed values for a node. + + **See also** :ref:`config-schema_object-validate` .. _config-schema_node_annotation-apply_default_if: - ``apply_default_if`` - A function that specifies whether to apply the default value specified using ``default``. + A boolean function that defines whether to apply the default value specified + using ``default``. If this function returns ``true`` on a provided configuration data, + the node receives the default value upon the :ref:`config-schema_object-apply_default` + method call. + + The function takes two arguments: + + - ``data`` -- the configuration data + - ``w`` -- *walkthrough node* with the following fields: + + - ``w.schema`` -- schema node + - ``w.path`` -- the path to the schema node + - ``w.error()` -- a function for printing human-readable error messages - See also: :ref:`config-schema_object-apply_default` + **See also:** :ref:`config-schema_object-apply_default`. .. _config-schema_node_annotation-default: - ``default`` A default value to use for a scalar node if it's not specified explicitly. - Example: :ref:`config_utils_schema_transform_configuration` - See also: :ref:`config-schema_object-apply_default` + **Example:** see :ref:`config_utils_schema_transform_configuration` + + **See also:** :ref:`config-schema_object-apply_default` .. _config-schema_node_annotation-type: - ``type`` - A value type. See also :ref:`config_utils_schema_data_types`. + A value type. + + **See also:** :ref:`config_utils_schema_data_types` .. _config-schema_node_annotation-validate: - ``validate`` - A function used to validate node data. + A boolean function used to validate node data. Node data is valid if this function + returns ``true``. The function is called upon the :ref:`config-schema_object-validate` + function calls. + + The function takes two arguments: + + - ``data`` -- the configuration data + - ``w`` -- *walkthrough node* with the following fields: - The function should accept the following arguments: + - ``w.schema`` -- schema node + - ``w.path`` -- the path to the schema node + - ``w.error()` -- a function for printing human-readable error messages See also :ref:`config-schema_object-validate` - Example: + **Example:** .. code-block:: lua @@ -991,13 +1088,15 @@ schema_node_object .. data:: allowed_values - A list of values allowed for the node. + A list of values allowed for the node. The values are taken from the + :ref:`config-schema_node_annotation-allowed_values` node annotation. .. _config-schema_node_object-apply_default_if: .. data:: apply_default_if - A + A function to define when to apply the default node value. The value + is taken from the :ref:`config-schema_node_annotation-apply_default_if` annotation. .. _config-schema_node_object-computed: @@ -1009,7 +1108,8 @@ schema_node_object .. data:: default - Node's default value. + Node's default value. The value + is taken from the :ref:`config-schema_node_annotation-default` annotation. .. _config-schema_node_object-fields: @@ -1035,4 +1135,5 @@ schema_node_object .. data:: validate - Node value validation function. + Node value validation function. The value + is taken from the :ref:`config-schema_node_annotation-validate` annotation. From e71f41e995ec08a781c858c94fbe99d1b32a4cfd Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Tue, 15 Oct 2024 18:36:25 +0700 Subject: [PATCH 21/33] Add text --- .../reference_lua/config/utils_schema.rst | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 3d39559d04..daa5140f79 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -867,12 +867,44 @@ schema_object .. method:: map(data, f, f_ctx) - Transform data by the given function. + Transform data by the given function. The ``data`` fields are transformed + by the function passed in the second argument (``f``), while its structure remains unchanged. + + The transformation function takes three arguments: + + - ``data`` -- the configuration data + - ``w`` -- *walkthrough node* with the following fields: + + - ``w.schema`` -- schema node + - ``w.path`` -- the path to the schema node + - ``w.error()` -- a function for printing human-readable error messages + + - ``ctx`` -- additional *context* for the transformation function. Can be + used to provide values for a specific call. + + An example of the transformation function: + + .. code-block:: lua + + local function f(data, w, ctx) + if w.schema.type == 'string' and data ~= nil then + return data:gsub('{{ *foo *}}', ctx.foo) + end + return data + end + + The ``map()`` method traverses all fields of the schema records, + even if they are ``nil`` or ``box.NULL`` in the provided configuration. + This allows using this method to set computed default values for missing + fields. Note that this is not the case for maps and arrays since the schema + doesn't define their fields to traverse. :param any data: value at the given path :param any f: walkthrough node, described below :param any f_ctx: user-provided context for the transformation function + :return: transformed configuration data + .. _config-schema_object-merge: .. method:: merge(data_a, data_b) From 405e847247a3231f531e5c64a9b30c6abc48091b Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Tue, 15 Oct 2024 19:05:40 +0700 Subject: [PATCH 22/33] Fix --- .../reference_lua/config/utils_schema.rst | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index daa5140f79..99ec4b4858 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -199,7 +199,7 @@ three scalar fields: :start-at: roles :dedent: -To define a record node in a schema, use :ref:`schema.record() `: +To define a record node in a schema, use :ref:`schema.record() `. The following schema describes the configuration above: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record/http_api.lua @@ -314,11 +314,11 @@ Built-in annotations Built-in annotations are interpreted by the module itself. There are the following built-in annotations: -- :ref:`type ` -- the node value type. Mandatory for scalar nodes, except for those created with ``schema.enum()``. -- :ref:`allowed_values ` -- a list of possible node values. -- :ref:`validate ` -- a validation function for the provided node value. -- :ref:`default ` -- a value to use if the option is not specified in the configuration. -- :ref:`apply_default_if ` -- a function that defines when to apply the default value. +- :ref:`type ` -- the node value type. Mandatory for scalar nodes, except for those created with ``schema.enum()``. +- :ref:`allowed_values ` -- a list of possible node values. +- :ref:`validate ` -- a validation function for the provided node value. +- :ref:`default ` -- a value to use if the option is not specified in the configuration. +- :ref:`apply_default_if ` -- a function that defines when to apply the default value. Consider the following role configuration: @@ -468,7 +468,7 @@ Transforming configuration The module provides methods that transform configuration data based on the schema, for example, :ref:`apply_default() `, -:ref:`merge() `, ref:`set() `. +:ref:`merge() `, :ref:`set() `. The following sample shows how to apply default values from the schema to fill missing configuration fields: @@ -485,9 +485,9 @@ missing configuration fields: Parsing environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`schema.fromenv() ` allows getting configuration -values from environment variables. The example below shows how to do this by -adding a user-defined annotation ``env``: +The :ref:`schema.fromenv() ` function allows getting +configuration values from environment variables. The example below shows how to do +this by adding a user-defined annotation ``env``: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_fromenv/http_api.lua :language: lua @@ -674,12 +674,14 @@ Functions How the raw value is parsed depends on the ``schema_node`` type: - Scalar: + - ``string``: return the value as is - - ``number`` or ``integer: parse the value as a number or an integer + - ``number`` or ``integer``: parse the value as a number or an integer - ``string, number``: attempt to parse as a number; in case of a failure return the value as is - ``boolean``: accept ``true`` or ``1`` for ``true``, ``false`` or ``0`` for ``false`` - ``any``: parse the value as a JSON + - Map: parse either as JSON (if the raw value starts with ``{``) or as a comma-separated string of ``key=value`` pairs: ``key1=value1,key2=value2`` - Array: parse either as JSON (if the raw value starts with ``[``) @@ -804,7 +806,7 @@ schema_object **See also:** :ref:`config-schema_node_annotation-default`, :ref:`config-schema_node_annotation-apply_default_if` -.. _config-schema_object-filter: + .. _config-schema_object-filter: .. method:: filter(data, f) @@ -877,7 +879,7 @@ schema_object - ``w.schema`` -- schema node - ``w.path`` -- the path to the schema node - - ``w.error()` -- a function for printing human-readable error messages + - ``w.error()`` -- a function for printing human-readable error messages - ``ctx`` -- additional *context* for the transformation function. Can be used to provide values for a specific call. @@ -1046,7 +1048,7 @@ parsed by the modules as :ref:`built-in annotations Date: Wed, 16 Oct 2024 09:38:08 +0700 Subject: [PATCH 23/33] Fix --- .../application_role_cfg/greeter.lua | 15 ++++--- doc/platform/app/app_roles.rst | 27 ++++++------ .../configuration/configuration_reference.rst | 6 +++ .../reference_lua/config/utils_schema.rst | 42 ++++++++++--------- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/doc/code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua b/doc/code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua index 2be88cde8a..0ce468c661 100644 --- a/doc/code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua +++ b/doc/code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua @@ -1,15 +1,20 @@ -- greeter.lua -- local log = require('log').new("greeter") +local schema = require('experimental.config.utils.schema') + +local greeter_schema = schema.new('greeter', schema.record({ + greeting = schema.scalar({ + type = 'string', + allowed_values = { 'Hi', 'Hello' } + }) +})) local function validate(cfg) - if cfg.greeting then - assert(type(cfg.greeting) == "string", "'greeting' should be a string") - assert(cfg.greeting == "Hi" or cfg.greeting == "Hello", "'greeting' should be 'Hi' or 'Hello'") - end + greeter_schema:validate(cfg) end local function apply(cfg) - log.info("%s from the 'greeter' role!", cfg.greeting) + log.info("%s from the 'greeter' role!", greeter_schema:get(cfg, 'greeting')) end local function stop() diff --git a/doc/platform/app/app_roles.rst b/doc/platform/app/app_roles.rst index 96b6bdcbb4..418a2a2cb3 100644 --- a/doc/platform/app/app_roles.rst +++ b/doc/platform/app/app_roles.rst @@ -27,8 +27,6 @@ To learn how to enable and configure roles, see :ref:`configuration_application_ - A role of a replica set in regard to sharding. Learn more in :ref:`vshard_config_sharding_roles`. - - .. _roles_create_custom_role_config: Providing a role configuration @@ -44,10 +42,15 @@ This example shows how to enable and configure the ``greeter`` role, which is im :start-at: instance001 :dedent: -The role's configuration provided in ``roles_cfg`` can be accessed when :ref:`validating ` and :ref:`applying ` this configuration. +The role configuration provided in ``roles_cfg`` can be accessed when :ref:`validating ` and :ref:`applying ` this configuration. -Given that a role is a :ref:`Lua module `, a role's name is passed to ``require()`` to obtain the module. -When :ref:`developing an application `, you can place a file with a role's code next to a cluster's configuration file. +Tarantool includes the :ref:`experimental.config.utils.schema ` +built-in module that provides an for managing user-defined configurations +of applications (``app.cfg``) and roles (``roles_cfg``). The examples below show its +basic usage. + +Given that a role is a :ref:`Lua module `, a role name is passed to ``require()`` to obtain the module. +When :ref:`developing an application `, you can place a file with the role code next to the cluster configuration file. @@ -63,12 +66,12 @@ Overview Creating a custom role includes the following steps: -1. Define a function that validates a role's configuration. +1. Define a function that validates a role configuration. 2. Define a function that applies a validated configuration. 3. Define a function that stops a role. 4. (Optional) Define roles from which this custom role depends on. -As a result, a role's module should return an object that has corresponding functions and fields specified: +As a result, a role module should return an object that has corresponding functions and fields specified: .. code-block:: lua @@ -86,14 +89,12 @@ The examples below show how to do this. Code snippets shown in this section are included from the following application: `application_role_cfg `_. - - .. _roles_create_custom_role_validate: Validating a role configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To validate a role's configuration, you need to define the :ref:`validate([cfg]) ` function. +To validate a role configuration, you need to define the :ref:`validate([cfg]) ` function. The ``cfg`` argument provides access to the :ref:`role's configuration ` and check its validity. In the example below, the ``validate()`` function is used to validate the ``greeting`` configuration value: @@ -107,8 +108,6 @@ In the example below, the ``validate()`` function is used to validate the ``gree If the configuration is not valid, ``validate()`` reports an unrecoverable error by throwing an error object. - - .. _roles_create_custom_role_apply: Applying a role configuration @@ -117,7 +116,7 @@ Applying a role configuration To apply the validated configuration, define the :ref:`apply([cfg]) ` function. As the ``validate()`` function, ``apply()`` provides access to a role's configuration using the ``cfg`` argument. -In the example below, the ``apply()`` function uses the :ref:`log ` module to write a role's configuration value to the log: +In the example below, the ``apply()`` function uses the :ref:`log ` module to write a value from the role configuration to the log: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua :language: lua @@ -142,7 +141,7 @@ In the example below, the ``stop()`` function uses the :ref:`log ` m :end-before: return :dedent: -When you've defined all the role's functions, you need to return an object that has corresponding functions specified: +When you've defined all the role functions, you need to return an object that has corresponding functions specified: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua :language: lua diff --git a/doc/reference/configuration/configuration_reference.rst b/doc/reference/configuration/configuration_reference.rst index 79a427bbdd..dbd84c63cd 100644 --- a/doc/reference/configuration/configuration_reference.rst +++ b/doc/reference/configuration/configuration_reference.rst @@ -46,6 +46,12 @@ In the ``app`` section, you can load the application and provide an application Example on GitHub: `application `_ + .. tip:: + + The :ref:`experimental.config.utils.schema ` + built-in module provides an API for managing user-defined configurations + of applications (``app.cfg``) and roles (``roles_cfg``). + | | Type: map | Default: nil diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 99ec4b4858..76560f05ee 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -52,7 +52,7 @@ the role configuration: :end-before: local function validate :dedent: -3. Use the :ref:`schema.validate() ` function to +3. Use the :ref:`validate() ` function to validate configuration values against the schema. In case of a role, call this function inside the role's :ref:`validate() ` function: @@ -62,7 +62,7 @@ the role configuration: :end-before: local function apply :dedent: -4. Obtain values of configuration options using :ref:`schema.get() `. +4. Obtain values of configuration options using :ref:`get() `. In case of a role, call it inside the role's :ref:`apply() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua @@ -804,7 +804,7 @@ schema_object :return: configuration data with applied schema defaults - **See also:** :ref:`config-schema_node_annotation-default`, :ref:`config-schema_node_annotation-apply_default_if` + **See also:** :ref:`default `, :ref:`apply_default_if ` .. _config-schema_object-filter: @@ -905,7 +905,7 @@ schema_object :param any f: walkthrough node, described below :param any f_ctx: user-provided context for the transformation function - :return: transformed configuration data + :return: transformed configuration data .. _config-schema_object-merge: @@ -981,18 +981,19 @@ schema_object The method performs the following checks: - field type checks: field values are checked against the schema node types - - allowed values: if a node has the :ref:`config-schema_node_annotation-allowed_values` - annotations of schema nodes, the corresponding data field is checked - against the allowed values list + - allowed values: if a node has the ``allowed_values`` annotations of schema nodes, + the corresponding data field is checked against the allowed values list - validation functions: if a validation function is defined for a node - (the :ref:`config-schema_node_annotation-validate` annotation), it is - executed to check that the provided value is valid. + (the ``validate`` annotation), it is executed to check that the provided value is valid. :param any data: data **Example:** see :ref:`config_utils_schema_annotation` and :ref:`config_utils_schema_validating_configuration` + **See also:** :ref:`allowed_values `, + :ref:`validate ` + .. _config-schema_object-methods: .. data:: methods @@ -1030,7 +1031,8 @@ parsed by the modules as :ref:`built-in annotations ` + .. _config-schema_node_annotation-apply_default_if: @@ -1038,7 +1040,7 @@ parsed by the modules as :ref:`built-in annotations ` method call. The function takes two arguments: @@ -1050,7 +1052,7 @@ parsed by the modules as :ref:`built-in annotations ` .. _config-schema_node_annotation-default: @@ -1060,7 +1062,7 @@ parsed by the modules as :ref:`built-in annotations ` .. _config-schema_node_annotation-type: @@ -1075,7 +1077,7 @@ parsed by the modules as :ref:`built-in annotations ` function calls. The function takes two arguments: @@ -1087,8 +1089,6 @@ parsed by the modules as :ref:`built-in annotations ` + .. _config-utils-schema_node_object: schema_node_object @@ -1111,14 +1113,14 @@ schema_node_object .. data:: allowed_values A list of values allowed for the node. The values are taken from the - :ref:`config-schema_node_annotation-allowed_values` node annotation. + :ref:`allowed_values ` node annotation. .. _config-schema_node_object-apply_default_if: .. data:: apply_default_if A function to define when to apply the default node value. The value - is taken from the :ref:`config-schema_node_annotation-apply_default_if` annotation. + is taken from the :ref:`apply_default_if ` annotation. .. _config-schema_node_object-computed: @@ -1131,7 +1133,7 @@ schema_node_object .. data:: default Node's default value. The value - is taken from the :ref:`config-schema_node_annotation-default` annotation. + is taken from the :ref:`default ` annotation. .. _config-schema_node_object-fields: @@ -1158,4 +1160,4 @@ schema_node_object .. data:: validate Node value validation function. The value - is taken from the :ref:`config-schema_node_annotation-validate` annotation. + is taken from the :ref:`validate ` annotation. From 36cb3c3a7e26076426e414fce10e43200aac92f1 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Wed, 16 Oct 2024 10:51:35 +0700 Subject: [PATCH 24/33] Fix --- doc/reference/configuration/configuration_reference.rst | 6 ++++++ doc/reference/reference_lua/config/utils_schema.rst | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/reference/configuration/configuration_reference.rst b/doc/reference/configuration/configuration_reference.rst index dbd84c63cd..1c569a3acb 100644 --- a/doc/reference/configuration/configuration_reference.rst +++ b/doc/reference/configuration/configuration_reference.rst @@ -3912,6 +3912,12 @@ This section describes configuration parameters related to :ref:`application rol See also: :ref:`configuration_application_roles` + .. tip:: + + The :ref:`experimental.config.utils.schema ` + built-in module provides an API for managing user-defined configurations + of applications (``app.cfg``) and roles (``roles_cfg``). + | | Type: map | Default: nil diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 76560f05ee..83e59c9537 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -413,7 +413,7 @@ User-defined methods A schema can implement custom logic with *methods* -- user-defined functions that can be called on this schema. -For example, this schema has a method that returns its fields merged in a URI string: +For example, this schema has the ``format`` method that returns its fields merged in a URI string: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_methods/http_api.lua :language: lua @@ -451,7 +451,7 @@ When writing roles, call this function inside the :ref:`role validation function Getting configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To get configuration values, use the schema's :ref:`get() ` method. +To get configuration values, use the schema object's :ref:`get() ` method. The function takes the configuration and the full path to the node as arguments: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua @@ -941,7 +941,7 @@ schema_object .. important:: - The method doesn't return records nodes. + The method doesn't return record nodes. :return: a luafun iterator @@ -1014,7 +1014,7 @@ schema_object Schema nodes hierarchy. - **See also:** ``schema_node_object`` + **See also:** :ref:`config-utils-schema_node_object` .. _config-utils-schema_node_annotation: From cbc4bcdbebd7b5ba671799f0552f00370efaca31 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Wed, 16 Oct 2024 11:39:01 +0700 Subject: [PATCH 25/33] Fix --- doc/platform/app/app_roles.rst | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/doc/platform/app/app_roles.rst b/doc/platform/app/app_roles.rst index 418a2a2cb3..ac5eca26cc 100644 --- a/doc/platform/app/app_roles.rst +++ b/doc/platform/app/app_roles.rst @@ -66,10 +66,11 @@ Overview Creating a custom role includes the following steps: -1. Define a function that validates a role configuration. -2. Define a function that applies a validated configuration. -3. Define a function that stops a role. -4. (Optional) Define roles from which this custom role depends on. +#. (Optional) Define the role configuration schema. +#. Define a function that validates a role configuration. +#. Define a function that applies a validated configuration. +#. Define a function that stops a role. +#. (Optional) Define roles from which this custom role depends on. As a result, a role module should return an object that has corresponding functions and fields specified: @@ -88,6 +89,26 @@ The examples below show how to do this. Code snippets shown in this section are included from the following application: `application_role_cfg `_. +.. _roles_create_custom_role_schema: + +Defining the role configuration schema +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`experimental.config.utils.schema ` built-in module +provides the :ref:`config-utils-schema_object` class. An object of this class defines +a custom configuration scheme of a role or an application. + +This example shows how to define a scheme that reflect the role configuration shown above: + +.. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua + :language: lua + :start-at: local greeter_schema + :end-before: local function validate + :dedent: + +If you don't use the module, skip this step. In this case, use the ``cfg`` argument +of the role's ``validate()`` and ``apply()`` function to refer to its configuration +values, for example, ``cfg.greeting``. .. _roles_create_custom_role_validate: @@ -97,7 +118,8 @@ Validating a role configuration To validate a role configuration, you need to define the :ref:`validate([cfg]) ` function. The ``cfg`` argument provides access to the :ref:`role's configuration ` and check its validity. -In the example below, the ``validate()`` function is used to validate the ``greeting`` configuration value: +In the example below, the ``validate()`` function of the role configuration schema +is used to validate the ``greeting`` value: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua :language: lua From f3e0c74663bab808ff897d2ef9d12abf2d998873 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Wed, 16 Oct 2024 11:54:33 +0700 Subject: [PATCH 26/33] Fix --- doc/platform/app/app_roles.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/platform/app/app_roles.rst b/doc/platform/app/app_roles.rst index ac5eca26cc..281aa47575 100644 --- a/doc/platform/app/app_roles.rst +++ b/doc/platform/app/app_roles.rst @@ -98,7 +98,7 @@ The :ref:`experimental.config.utils.schema ` built-i provides the :ref:`config-utils-schema_object` class. An object of this class defines a custom configuration scheme of a role or an application. -This example shows how to define a scheme that reflect the role configuration shown above: +This example shows how to define a schema that reflects the role configuration shown above: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/application_role_cfg/greeter.lua :language: lua @@ -107,7 +107,7 @@ This example shows how to define a scheme that reflect the role configuration sh :dedent: If you don't use the module, skip this step. In this case, use the ``cfg`` argument -of the role's ``validate()`` and ``apply()`` function to refer to its configuration +of the role's ``validate()`` and ``apply()`` functions to refer to its configuration values, for example, ``cfg.greeting``. .. _roles_create_custom_role_validate: From 76125f69acff7637beaeefd2e147bc4fdbd6680b Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Thu, 17 Oct 2024 12:30:12 +0700 Subject: [PATCH 27/33] Fix --- .../reference_lua/config/utils_schema.rst | 153 ++++++++++++------ 1 file changed, 105 insertions(+), 48 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 83e59c9537..c507c412e7 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -52,9 +52,9 @@ the role configuration: :end-before: local function validate :dedent: -3. Use the :ref:`validate() ` function to +3. Use the :ref:`validate() ` method of the schema object to validate configuration values against the schema. In case of a role, call this - function inside the role's :ref:`validate() ` function: + method inside the role's :ref:`validate() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua :language: lua @@ -62,7 +62,7 @@ the role configuration: :end-before: local function apply :dedent: -4. Obtain values of configuration options using :ref:`get() `. +4. Obtain values of configuration options using the :ref:`get() ` method. In case of a role, call it inside the role's :ref:`apply() ` function: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua @@ -85,10 +85,10 @@ option names, types, hierarchy, and other aspects of a configuration. To create a schema, use the :ref:`schema.new() ` function. It has the following arguments: -- schema name -- an arbitrary string to use as an identifier -- root schema node -- a table describing the hierarchical schema structure +- Schema name -- an arbitrary string to use as an identifier. +- Root schema node -- a table describing the hierarchical schema structure starting from the root. -- (optional) methods -- functions that can be called on this schema object. +- (Optional) methods -- user-defined functions that can be called on this schema object. .. _config_utils_schema_nodes: @@ -287,7 +287,8 @@ Annotations Node *annotations* are named attributes that define the its various aspects. For example, scalar nodes have a required annotation ``type`` that defines the node value type. -Other annotations can optionally set default values, validation function, and +Other annotations can, for example, set a node's default value and a validation function, +or store arbitrary user-provided data. Annotations are passed in a table to the node creation function: @@ -314,11 +315,14 @@ Built-in annotations Built-in annotations are interpreted by the module itself. There are the following built-in annotations: -- :ref:`type ` -- the node value type. Mandatory for scalar nodes, except for those created with ``schema.enum()``. -- :ref:`allowed_values ` -- a list of possible node values. -- :ref:`validate ` -- a validation function for the provided node value. -- :ref:`default ` -- a value to use if the option is not specified in the configuration. -- :ref:`apply_default_if ` -- a function that defines when to apply the default value. +- :ref:`type ` -- the node value type. + The type must be explicitly specified for scalar nodes, except for those created with ``schema.enum()``. + For composite nodes and scalar enums, the corresponding constructors ``schema.record()``, ``schema.map()``, ``schema.array()``, + ``schema.set()``, and ``schema.enum`` set the type automatically. +- :ref:`allowed_values ` -- (optional) a list of possible node values . +- :ref:`validate ` -- (optional) a validation function for the provided node value. +- :ref:`default ` -- (optional) a value to use if the option is not specified in the configuration. +- :ref:`apply_default_if ` -- (optional) a function that defines when to apply the default value. Consider the following role configuration: @@ -433,9 +437,9 @@ Processing configuration data Validating configuration ~~~~~~~~~~~~~~~~~~~~~~~~ -The schema object's :ref:`validate() ` function performs all the necessary checks +The schema object's :ref:`validate() ` method performs all the necessary checks on the provided configuration. It validates the configuration structure, node types, allowed values, -and other aspects defines in the schema. +and other aspects of the schema. When writing roles, call this function inside the :ref:`role validation function `: @@ -452,7 +456,7 @@ Getting configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To get configuration values, use the schema object's :ref:`get() ` method. -The function takes the configuration and the full path to the node as arguments: +It takes the configuration and the full path to the node as arguments: .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_record_hierarchy/http_api.lua :language: lua @@ -466,7 +470,7 @@ The function takes the configuration and the full path to the node as arguments: Transforming configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The module provides methods that transform configuration data based on the schema, +The schema object has methods that transform configuration data based on the schema, for example, :ref:`apply_default() `, :ref:`merge() `, :ref:`set() `. @@ -518,28 +522,28 @@ API Reference - * - :ref:`schema.array() ` - - Create an array + - Create an array node * - :ref:`schema.enum() ` - - Create an enum + - Create an enum scalar node * - :ref:`schema.fromenv() ` - Parse a value from an environment variable * - :ref:`schema.map() ` - - Create a map + - Create a map node * - :ref:`schema.new() ` - Create a schema * - :ref:`schema.record() ` - - Create a record + - Create a record node * - :ref:`schema.scalar() ` - - Create a scalar + - Create a scalar node * - :ref:`schema.set() ` - - Create a set + - Create a set array node * - **schema_object** - @@ -632,17 +636,20 @@ Functions .. _config-utils-schema-array: -.. function:: schema.array(opts) +.. function:: schema.array(array_def) Create an array node of a configuration schema. - :param table opts: a table in the following format: + :param table array_def: a table in the following format: - .. code-block:: text + .. code-block:: text - { items = , <..annotations..> } + { + items = , + <..annotations..> + } - See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. :return: the created schema node :rtype: table @@ -679,7 +686,8 @@ Functions - ``number`` or ``integer``: parse the value as a number or an integer - ``string, number``: attempt to parse as a number; in case of a failure return the value as is - - ``boolean``: accept ``true`` or ``1`` for ``true``, ``false`` or ``0`` for ``false`` + - ``boolean``: accept ``true`` and ``false`` (case-insensitively), or ``1``and ``0`` + for ``true`` and ``false`` values correspondingly - ``any``: parse the value as a JSON - Map: parse either as JSON (if the raw value starts with ``{``) @@ -687,28 +695,36 @@ Functions - Array: parse either as JSON (if the raw value starts with ``[``) or as a comma-separated string of items: ``item1,item2,item3`` - :param string env_var_name: environment variable name + .. note:: + + Parsing records from environment variables is not supported. + + :param string env_var_name: environment variable name to use for error messages :param string raw_value: environment variable value :param schema_node_object schema_node: a schema node (see :ref:`schema_node_object `) - :return: the created schema node + :return: the parsed value :rtype: table **See also:** :ref:`config_utils_schema_env-vars` .. _config-utils-schema-map: -.. function:: schema.map(opts) +.. function:: schema.map(map_def) Create a map node of a configuration schema. - :param table opts: a table in the following format: + :param table map_def: a table in the following format: - .. code-block:: text + .. code-block:: text - { key = , value = , <..annotations..> } + { + key = , + value = , + <..annotations..> + } - See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. :return: the created schema node :rtype: table @@ -725,8 +741,14 @@ Functions :param table schema_node: a root schema node :param table methods: methods - :return: a new schema instance (see :ref:`schema_object `) - :rtype: userdata + :return: a new schema object (see :ref:`schema_object `) + as a table with the following fields: + + * ``name``: the schema name + * ``schema``: a table with schema nodes + * ``methods``: a table with user-provided methods + + :rtype: table **See also:** :ref:`config_utils_schema_getting_started` @@ -740,7 +762,10 @@ Functions .. code-block:: text - { [] = , <...> } + { + [] = , + <...> + } See also: :ref:`schema_node_object `. @@ -790,6 +815,10 @@ schema_object .. method:: apply_default(data) + .. important:: + + ``data`` is assumed to be validated against the given schema. + Apply default values to scalar nodes. The functions takes the ``default`` built-in annotation values of the scalar nodes and applies them based on the ``apply_default_if`` annotation. If there is no ``apply_default_if`` @@ -810,7 +839,11 @@ schema_object .. method:: filter(data, f) - Filter data based on the schema annotations. The methods returns an iterator + .. important:: + + ``data`` is assumed to be validated against the given schema. + + Filter data based on the schema annotations. The method returns an iterator by configuration nodes for which the given filter function ``f`` returns ``true``. The filter function ``f`` receives the following table as the argument: @@ -847,6 +880,10 @@ schema_object .. method:: get(data, path) + .. important:: + + ``data`` is assumed to be validated against the given schema. + Get nested configuration values at the given path. The path can be either a dot-separated string (``http.scheme``) or an array-like table (``{ 'http', 'scheme'}``). @@ -859,7 +896,9 @@ schema_object :dedent: :param any data: configuration data - :param any path: path to the target node + :param string path: path to the target node in the dot notation + :param table path: path to the target node in an array-like table + :return: data at the given path @@ -869,6 +908,10 @@ schema_object .. method:: map(data, f, f_ctx) + .. important:: + + ``data`` is assumed to be validated against the given schema. + Transform data by the given function. The ``data`` fields are transformed by the function passed in the second argument (``f``), while its structure remains unchanged. @@ -901,8 +944,8 @@ schema_object fields. Note that this is not the case for maps and arrays since the schema doesn't define their fields to traverse. - :param any data: value at the given path - :param any f: walkthrough node, described below + :param any data: configuration data + :param function f: transformation function :param any f_ctx: user-provided context for the transformation function :return: transformed configuration data @@ -911,6 +954,10 @@ schema_object .. method:: merge(data_a, data_b) + .. important:: + + ``data`` is assumed to be validated against the given schema. + Merge two configurations. The method merges configurations in a single node hierarchy, preferring the latter in case of a collision. @@ -922,8 +969,12 @@ schema_object .. important:: - Note that the method doesn't concatenate arrays. Left hand array - items are discarded. + Scalars of the ``any`` type are merged the same way as other scalars. + They are not deeply merged even if they are tables. + + .. important:: + + Arrays are not concatenated. Left hand array items are discarded. - records and maps are deeply merged, that is, the merge is performed recursively for their nested nodes @@ -959,12 +1010,18 @@ schema_object .. method:: set(data, path, value) + .. important:: + + ``data`` is assumed to be validated against the given schema. + ``value`` is validated by the method before the assignment. + Set a given value at the given path in a configuration. The path can be either a dot-separated string (``http.scheme``) or an array-like table (``{ 'http', 'scheme'}``). :param any data: configuration data - :param any path: path to the target node + :param string path: path to the target node in the dot notation + :param table path: path to the target node in an array-like table :param any value: new value :return: updated configuration data @@ -1068,7 +1125,7 @@ parsed by the modules as :ref:`built-in annotations ` + A function used to validate node data. The function must raise an error to + fail the check. The function is called upon the :ref:`schema_object:validate() ` function calls. The function takes two arguments: From d026a3815aefa45cf973be4a78be486de07c47e5 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Thu, 17 Oct 2024 15:56:00 +0700 Subject: [PATCH 28/33] Add table structure to ref --- .../reference_lua/config/utils_schema.rst | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index c507c412e7..4118cb7c67 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -651,7 +651,12 @@ Functions See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. - :return: the created schema node + :return: the created array node as a table with the following fields: + + - ``type``: ``array`` + - ``items``: a table describing an array item as a schema node + - annotations, if provided in ``array_def`` + :rtype: table **See also:** :ref:`config_utils_schema_nodes_array` @@ -665,7 +670,12 @@ Functions :param table allowed_values: a list of enum members -- values allowed for the node :param table annotations: annotations (see :ref:`schema_node_annotation `) - :return: the created schema node + :return: the created scalar node as a table with the following fields: + + - ``type``: ``string`` + - ``allowed_values``: allowed node values + - annotations, if ``annotations`` is provided + :rtype: table **See also:** :ref:`config_utils_schema_nodes_scalar` @@ -726,7 +736,13 @@ Functions See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. - :return: the created schema node + :return: the created map node as a table with the following fields: + + - ``type``: ``map`` + - ``key``: map key type + - ``value``: map value type + - annotations, if provided in ``map_def`` + :rtype: table **See also:** :ref:`config_utils_schema_nodes_map` @@ -771,7 +787,12 @@ Functions :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: the created schema node + :return: the created record node as a table with the following fields: + + - ``type``: ``record`` + - ``fields``: a table describing the record fields + - annotations, if provided + :rtype: table **See also:** :ref:`config_utils_schema_nodes_record` @@ -785,7 +806,11 @@ Functions :param string type: data type (see :ref:`config_utils_schema_data_types`) :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: the created schema node + :return: the created scalar node as a table with the following fields: + + - ``type``: the node type (see :ref:`config_utils_schema_data_types`) + - annotations, if provided + :rtype: table **See also:** :ref:`config_utils_schema_nodes_scalar` From f83e7467e73cc87a956cd18ce205f709046f950f Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Fri, 18 Oct 2024 15:28:27 +0700 Subject: [PATCH 29/33] Fix --- .../reference_lua/config/utils_schema.rst | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 4118cb7c67..f395b03882 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -824,7 +824,14 @@ Functions :param table allowed_values: allowed values of array items :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) - :return: the created schema node + :return: the created array node as a table with the following fields: + + - ``type``: ``array`` + - ``items``: a table describing an array item as a schema node + - ``validate``: an auto-generated validation function that check + that the values don't repeat + - annotations, if provided + :rtype: table **See also:** :ref:`config_utils_schema_nodes_array` @@ -921,8 +928,9 @@ schema_object :dedent: :param any data: configuration data - :param string path: path to the target node in the dot notation - :param table path: path to the target node in an array-like table + :param string/table path: path to the target node as: + - a string in the dot notation + - an array-like table :return: data at the given path @@ -992,14 +1000,11 @@ schema_object - ``box.NULL`` is preferred over ``nil`` - for scalar and array nodes, the right-hand value is used - .. important:: - - Scalars of the ``any`` type are merged the same way as other scalars. - They are not deeply merged even if they are tables. - - .. important:: + .. note:: - Arrays are not concatenated. Left hand array items are discarded. + - Scalars of the ``any`` type are merged the same way as other scalars. + They are not deeply merged even if they are tables. + - Arrays are not concatenated. Left hand array items are discarded. - records and maps are deeply merged, that is, the merge is performed recursively for their nested nodes @@ -1013,7 +1018,7 @@ schema_object .. method:: pairs() - Walk over the schema and return scalar, array and map schema nodes + Walk over the schema and return scalar, array, and map schema nodes .. important:: @@ -1045,8 +1050,9 @@ schema_object an array-like table (``{ 'http', 'scheme'}``). :param any data: configuration data - :param string path: path to the target node in the dot notation - :param table path: path to the target node in an array-like table + :param string/table path: path to the target node as: + - a string in the dot notation + - an array-like table :param any value: new value :return: updated configuration data From 47c7f975b9c35da2b57250b21755b97b1d75eba6 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Fri, 18 Oct 2024 16:26:47 +0700 Subject: [PATCH 30/33] Fix --- .../reference_lua/config/utils_schema.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index f395b03882..f143307990 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -52,6 +52,8 @@ the role configuration: :end-before: local function validate :dedent: + Learn more in :ref:`config_utils_schema_definition`. + 3. Use the :ref:`validate() ` method of the schema object to validate configuration values against the schema. In case of a role, call this method inside the role's :ref:`validate() ` function: @@ -62,14 +64,12 @@ the role configuration: :end-before: local function apply :dedent: -4. Obtain values of configuration options using the :ref:`get() ` method. - In case of a role, call it inside the role's :ref:`apply() ` function: + Learn more in :ref:`config_utils_schema_validating_configuration`. + +4. Refer to values of configuration options using the :ref:`get() ` + method inside the role's :ref:`apply() ` function. + Learn more in :ref:`config_utils_schema_get_configuration`. - .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/http_api.lua - :language: lua - :start-at: local function apply - :end-before: local function stop - :dedent: .. _config_utils_schema_definition: From bee7bf5a045bd870c88d5e4b40b7520d47cc3a9b Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Fri, 18 Oct 2024 17:06:30 +0700 Subject: [PATCH 31/33] Fix --- .../reference_lua/config/utils_schema.rst | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index f143307990..560f89125f 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -318,7 +318,7 @@ built-in annotations: - :ref:`type ` -- the node value type. The type must be explicitly specified for scalar nodes, except for those created with ``schema.enum()``. For composite nodes and scalar enums, the corresponding constructors ``schema.record()``, ``schema.map()``, ``schema.array()``, - ``schema.set()``, and ``schema.enum`` set the type automatically. + ``schema.set()``, and ``schema.enum()`` set the type automatically. - :ref:`allowed_values ` -- (optional) a list of possible node values . - :ref:`validate ` -- (optional) a validation function for the provided node value. - :ref:`default ` -- (optional) a value to use if the option is not specified in the configuration. @@ -696,7 +696,7 @@ Functions - ``number`` or ``integer``: parse the value as a number or an integer - ``string, number``: attempt to parse as a number; in case of a failure return the value as is - - ``boolean``: accept ``true`` and ``false`` (case-insensitively), or ``1``and ``0`` + - ``boolean``: accept ``true`` and ``false`` (case-insensitively), or ``1`` and ``0`` for ``true`` and ``false`` values correspondingly - ``any``: parse the value as a JSON @@ -799,10 +799,21 @@ Functions .. _config-utils-schema-scalar: -.. function:: schema.scalar(type[, annotations]) +.. function:: schema.scalar(scalar_def) Create a scalar node of a configuration schema. + :param table scalar_def: a table in the following format: + + .. code-block:: text + + { + type = , + <..annotations..> + } + + See also: :ref:`schema_node_object `, :ref:`schema_node_annotation `. + :param string type: data type (see :ref:`config_utils_schema_data_types`) :param table annotations: annotations (see :ref:`config_utils_schema_annotation`) @@ -989,7 +1000,7 @@ schema_object .. important:: - ``data`` is assumed to be validated against the given schema. + ``data_a`` and ``data_b`` are assumed to be validated against the given schema. Merge two configurations. The method merges configurations in a single node hierarchy, preferring the latter in case of a collision. @@ -1009,8 +1020,8 @@ schema_object - records and maps are deeply merged, that is, the merge is performed recursively for their nested nodes - :param any a: configuration data - :param any b: configuration data + :param any data_a: configuration data + :param any data_b: configuration data :return: merged configuration data From 0761e11620f1cba48f616284729b8f86f0fab890 Mon Sep 17 00:00:00 2001 From: Pavel Semyonov Date: Tue, 22 Oct 2024 10:52:50 +0700 Subject: [PATCH 32/33] Apply suggestions from code review Co-authored-by: Elena Shebunyaeva --- doc/platform/app/app_roles.rst | 2 +- doc/reference/reference_lua/config/utils_schema.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/platform/app/app_roles.rst b/doc/platform/app/app_roles.rst index 281aa47575..bea58ef98d 100644 --- a/doc/platform/app/app_roles.rst +++ b/doc/platform/app/app_roles.rst @@ -45,7 +45,7 @@ This example shows how to enable and configure the ``greeter`` role, which is im The role configuration provided in ``roles_cfg`` can be accessed when :ref:`validating ` and :ref:`applying ` this configuration. Tarantool includes the :ref:`experimental.config.utils.schema ` -built-in module that provides an for managing user-defined configurations +built-in module that provides tools for managing user-defined configurations of applications (``app.cfg``) and roles (``roles_cfg``). The examples below show its basic usage. diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index 560f89125f..e3e1a1bd68 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -24,7 +24,7 @@ Getting started with config.utils.schema ---------------------------------------- As an example, consider an :ref:`application role ` -that has a single configuration option - an HTTP endpoint address. +that has a single configuration option -- an HTTP endpoint address. .. literalinclude:: /code_snippets/snippets/config/instances.enabled/config_schema_nodes_scalar/config.yaml :language: yaml @@ -285,7 +285,7 @@ any number of options with arbitrary names and boolean values. Annotations *********** -Node *annotations* are named attributes that define the its various aspects. For example, +Node *annotations* are named attributes that define its various aspects. For example, scalar nodes have a required annotation ``type`` that defines the node value type. Other annotations can, for example, set a node's default value and a validation function, or store arbitrary user-provided data. @@ -301,7 +301,7 @@ Annotations are passed in a table to the node creation function: Node annotations fall into three groups: - *Built-in annotations* are handled by the module. These are: ``type``, ``validate``, ``allowed_values``, ``default`` and ``apply_default_if``. - Note that ``validate``, ``allowed_values`` are used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. + Note that ``validate`` and ``allowed_values`` are used for validation only. ``default`` and ``apply_default_if`` can transform the configuration. - *User-defined annotations* add named node attributes that can be used in the application or role code. - *Computed annotations* allow access to annotations of other nodes throughout @@ -319,7 +319,7 @@ built-in annotations: The type must be explicitly specified for scalar nodes, except for those created with ``schema.enum()``. For composite nodes and scalar enums, the corresponding constructors ``schema.record()``, ``schema.map()``, ``schema.array()``, ``schema.set()``, and ``schema.enum()`` set the type automatically. -- :ref:`allowed_values ` -- (optional) a list of possible node values . +- :ref:`allowed_values ` -- (optional) a list of possible node values. - :ref:`validate ` -- (optional) a validation function for the provided node value. - :ref:`default ` -- (optional) a value to use if the option is not specified in the configuration. - :ref:`apply_default_if ` -- (optional) a function that defines when to apply the default value. @@ -839,7 +839,7 @@ Functions - ``type``: ``array`` - ``items``: a table describing an array item as a schema node - - ``validate``: an auto-generated validation function that check + - ``validate``: an auto-generated validation function that checks that the values don't repeat - annotations, if provided @@ -1190,7 +1190,7 @@ parsed by the modules as :ref:`built-in annotations Date: Tue, 22 Oct 2024 10:56:56 +0700 Subject: [PATCH 33/33] Review fixes --- doc/platform/app/app_roles.rst | 1 - doc/reference/reference_lua/config/utils_schema.rst | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/platform/app/app_roles.rst b/doc/platform/app/app_roles.rst index bea58ef98d..4d60a298c2 100644 --- a/doc/platform/app/app_roles.rst +++ b/doc/platform/app/app_roles.rst @@ -116,7 +116,6 @@ Validating a role configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To validate a role configuration, you need to define the :ref:`validate([cfg]) ` function. -The ``cfg`` argument provides access to the :ref:`role's configuration ` and check its validity. In the example below, the ``validate()`` function of the role configuration schema is used to validate the ``greeting`` value: diff --git a/doc/reference/reference_lua/config/utils_schema.rst b/doc/reference/reference_lua/config/utils_schema.rst index e3e1a1bd68..6292133a25 100644 --- a/doc/reference/reference_lua/config/utils_schema.rst +++ b/doc/reference/reference_lua/config/utils_schema.rst @@ -98,7 +98,7 @@ Schema nodes Schema nodes describe the hierarchy of options within a schema. There are two types of schema nodes: - *Scalar* nodes hold a single value of a supported primitive type. For example, - a string configuration option of a role is a scalar node its schema. + a string configuration option of a role is a scalar node in its schema. - *Composite* nodes include multiple values in different forms: *records*, *arrays*, or *maps*. A node can have *annotations* -- named attributes that enable customization of @@ -564,7 +564,7 @@ API Reference - Merge two configurations * - :ref:`schema_object:pairs() ` - - Traverse a configuration + - Walk over a configuration * - :ref:`schema_object:set() ` - Set a configuration value @@ -940,6 +940,7 @@ schema_object :param any data: configuration data :param string/table path: path to the target node as: + - a string in the dot notation - an array-like table @@ -1062,6 +1063,7 @@ schema_object :param any data: configuration data :param string/table path: path to the target node as: + - a string in the dot notation - an array-like table :param any value: new value