From 24edc804c94a0df8a7a8ddece377978882d5c364 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 13 Jul 2020 16:30:03 +0100 Subject: [PATCH] Node options from cfg file for production (#62468) * chore(NA): load NODE_OPTIONS from options files across environments * chore(NA): move node.ci.options to config folder * docs(NA): update docs to explain how to set node options from the cfg fil * chore(NA): removed test npm scripts * fix(NA): typo on setup script for CI * chore(NA): add debug info * chore(NA): export options on CI * chore(NA): remove debug info * chore(NA): support for configurable config folder using env var * chore(NA): add node.options file into docker img * fix(NA): use calculated config dir on node options for ci * chore(NA): node bin scripts bootstrap and node_with_options implementation for bash * chore(NA): complete node_with_options scripts with bat version * chore(NA): add bin/node dev script and remove cli for run_with_node_options * chore(NA): increase default maxBuffer * chore(NA): remove run with options script from package.json * chore(NA): include kbn-node script and underlying usage of it * chore(NA): remove change on eslint * chore(NA): correct typo on kbn node script comment Co-authored-by: Tyler Smalley * chore(NA): correct typo on kbn node script comment Co-authored-by: Tyler Smalley * chore(NA): add line to describe each option should be specified in a separated line * chore(NA): remove node options from dev and ci env * chore(NA): remove changes from package.json * chore(NA): fix docker image build * chore(NA): change value for example of --max-old-space-size in the node.options file Co-authored-by: Tyler Smalley * chore(NA): remove --no-warnings from node.options and force it in the bin scripts * chore(NA): prevent 'The system cannot find the file' error message * chore(NA): introduce slash when building path for %DIR% * chore(NA): read options from file only if it exists Co-authored-by: Jonathan Budzenski Co-authored-by: Elastic Machine Co-authored-by: Tyler Smalley --- .gitignore | 1 + config/node.options | 6 +++++ docs/setup/production.asciidoc | 4 ++-- package.json | 2 +- src/dev/build/tasks/bin/scripts/kibana | 7 +++++- .../build/tasks/bin/scripts/kibana-keystore | 7 +++++- .../tasks/bin/scripts/kibana-keystore.bat | 17 ++++++++++++- src/dev/build/tasks/bin/scripts/kibana-plugin | 7 +++++- .../build/tasks/bin/scripts/kibana-plugin.bat | 23 +++++++++++++++--- src/dev/build/tasks/bin/scripts/kibana.bat | 24 +++++++++++++++++-- src/dev/build/tasks/copy_source_task.js | 1 + 11 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 config/node.options diff --git a/.gitignore b/.gitignore index 716cea937f9c0..dfd02de7b1186 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ disabledPlugins webpackstats.json /config/* !/config/kibana.yml +!/config/node.options coverage selenium .babel_register_cache.json diff --git a/config/node.options b/config/node.options new file mode 100644 index 0000000000000..2927d1b576716 --- /dev/null +++ b/config/node.options @@ -0,0 +1,6 @@ +## Node command line options +## See `node --help` and `node --v8-options` for available options +## Please note you should specify one option per line + +## max size of old space in megabytes +#--max-old-space-size=4096 diff --git a/docs/setup/production.asciidoc b/docs/setup/production.asciidoc index 72f275e237490..afb4b37df6a28 100644 --- a/docs/setup/production.asciidoc +++ b/docs/setup/production.asciidoc @@ -167,9 +167,9 @@ These can be used to automatically update the list of hosts as a cluster is resi Kibana has a default maximum memory limit of 1.4 GB, and in most cases, we recommend leaving this unconfigured. In some scenarios, such as large reporting jobs, it may make sense to tweak limits to meet more specific requirements. -You can modify this limit by setting `--max-old-space-size` in the `NODE_OPTIONS` environment variable. For deb and rpm, packages this is passed in via `/etc/default/kibana` and can be appended to the bottom of the file. +You can modify this limit by setting `--max-old-space-size` in the `node.options` config file that can be found inside `kibana/config` folder or any other configured with the environment variable `KIBANA_PATH_CONF` (for example in debian based system would be `/etc/kibana`). The option accepts a limit in MB: -------- -NODE_OPTIONS="--max-old-space-size=2048" bin/kibana +--max-old-space-size=2048 -------- diff --git a/package.json b/package.json index 7889909b15244..7ab6bfb91a376 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "uiFramework:documentComponent": "cd packages/kbn-ui-framework && yarn documentComponent", "kbn:watch": "node scripts/kibana --dev --logging.json=false", "build:types": "tsc --p tsconfig.types.json", - "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept", + "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept", "kbn:bootstrap": "node scripts/register_git_hook", "spec_to_console": "node scripts/spec_to_console", "backport-skip-ci": "backport --prDescription \"[skip-ci]\"", diff --git a/src/dev/build/tasks/bin/scripts/kibana b/src/dev/build/tasks/bin/scripts/kibana index 558facb9da32b..3283e17008e7c 100755 --- a/src/dev/build/tasks/bin/scripts/kibana +++ b/src/dev/build/tasks/bin/scripts/kibana @@ -14,6 +14,7 @@ while [ -h "$SCRIPT" ] ; do done DIR="$(dirname "${SCRIPT}")/.." +CONFIG_DIR=${KIBANA_PATH_CONF:-"$DIR/config"} NODE="${DIR}/node/bin/node" test -x "$NODE" if [ ! -x "$NODE" ]; then @@ -21,4 +22,8 @@ if [ ! -x "$NODE" ]; then exit 1 fi -NODE_OPTIONS="--no-warnings --max-http-header-size=65536 ${NODE_OPTIONS}" NODE_ENV=production exec "${NODE}" "${DIR}/src/cli" ${@} +if [ -f "${CONFIG_DIR}/node.options" ]; then + KBN_NODE_OPTS="$(grep -v ^# < ${CONFIG_DIR}/node.options | xargs)" +fi + +NODE_OPTIONS="--no-warnings --max-http-header-size=65536 $KBN_NODE_OPTS $NODE_OPTIONS" NODE_ENV=production exec "${NODE}" "${DIR}/src/cli" ${@} diff --git a/src/dev/build/tasks/bin/scripts/kibana-keystore b/src/dev/build/tasks/bin/scripts/kibana-keystore index 43800c7b895d3..f83df118d24e8 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-keystore +++ b/src/dev/build/tasks/bin/scripts/kibana-keystore @@ -14,6 +14,7 @@ while [ -h "$SCRIPT" ] ; do done DIR="$(dirname "${SCRIPT}")/.." +CONFIG_DIR=${KIBANA_PATH_CONF:-"$DIR/config"} NODE="${DIR}/node/bin/node" test -x "$NODE" if [ ! -x "$NODE" ]; then @@ -21,4 +22,8 @@ if [ ! -x "$NODE" ]; then exit 1 fi -"${NODE}" "${DIR}/src/cli_keystore" "$@" +if [ -f "${CONFIG_DIR}/node.options" ]; then + KBN_NODE_OPTS="$(grep -v ^# < ${CONFIG_DIR}/node.options | xargs)" +fi + +NODE_OPTIONS="$KBN_NODE_OPTS $NODE_OPTIONS" "${NODE}" "${DIR}/src/cli_keystore" "$@" diff --git a/src/dev/build/tasks/bin/scripts/kibana-keystore.bat b/src/dev/build/tasks/bin/scripts/kibana-keystore.bat index b8311db2cfae5..389eb5bf488e4 100644 --- a/src/dev/build/tasks/bin/scripts/kibana-keystore.bat +++ b/src/dev/build/tasks/bin/scripts/kibana-keystore.bat @@ -1,6 +1,6 @@ @echo off -SETLOCAL +SETLOCAL ENABLEDELAYEDEXPANSION set SCRIPT_DIR=%~dp0 for %%I in ("%SCRIPT_DIR%..") do set DIR=%%~dpfI @@ -12,6 +12,21 @@ If Not Exist "%NODE%" ( Exit /B 1 ) +set CONFIG_DIR=%KIBANA_PATH_CONF% +If [%KIBANA_PATH_CONF%] == [] ( + set CONFIG_DIR=%DIR%\config +) + +IF EXIST "%CONFIG_DIR%\node.options" ( + for /F "eol=# tokens=*" %%i in (%CONFIG_DIR%\node.options) do ( + If [!NODE_OPTIONS!] == [] ( + set "NODE_OPTIONS=%%i" + ) Else ( + set "NODE_OPTIONS=!NODE_OPTIONS! %%i" + ) + ) +) + TITLE Kibana Keystore "%NODE%" "%DIR%\src\cli_keystore" %* diff --git a/src/dev/build/tasks/bin/scripts/kibana-plugin b/src/dev/build/tasks/bin/scripts/kibana-plugin index b843d4966c6d1..f1102e1ef5a32 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-plugin +++ b/src/dev/build/tasks/bin/scripts/kibana-plugin @@ -14,6 +14,7 @@ while [ -h "$SCRIPT" ] ; do done DIR="$(dirname "${SCRIPT}")/.." +CONFIG_DIR=${KIBANA_PATH_CONF:-"$DIR/config"} NODE="${DIR}/node/bin/node" test -x "$NODE" if [ ! -x "$NODE" ]; then @@ -21,4 +22,8 @@ if [ ! -x "$NODE" ]; then exit 1 fi -NODE_OPTIONS="--no-warnings ${NODE_OPTIONS}" NODE_ENV=production exec "${NODE}" "${DIR}/src/cli_plugin" "$@" +if [ -f "${CONFIG_DIR}/node.options" ]; then + KBN_NODE_OPTS="$(grep -v ^# < ${CONFIG_DIR}/node.options | xargs)" +fi + +NODE_OPTIONS="--no-warnings $KBN_NODE_OPTS $NODE_OPTIONS" NODE_ENV=production exec "${NODE}" "${DIR}/src/cli_plugin" "$@" diff --git a/src/dev/build/tasks/bin/scripts/kibana-plugin.bat b/src/dev/build/tasks/bin/scripts/kibana-plugin.bat index bf382a0657ade..6815b1b9eab8c 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-plugin.bat +++ b/src/dev/build/tasks/bin/scripts/kibana-plugin.bat @@ -1,6 +1,6 @@ @echo off -SETLOCAL +SETLOCAL ENABLEDELAYEDEXPANSION set SCRIPT_DIR=%~dp0 for %%I in ("%SCRIPT_DIR%..") do set DIR=%%~dpfI @@ -13,9 +13,26 @@ If Not Exist "%NODE%" ( Exit /B 1 ) -TITLE Kibana Server +set CONFIG_DIR=%KIBANA_PATH_CONF% +If [%KIBANA_PATH_CONF%] == [] ( + set CONFIG_DIR=%DIR%\config +) + +IF EXIST "%CONFIG_DIR%\node.options" ( + for /F "eol=# tokens=*" %%i in (%CONFIG_DIR%\node.options) do ( + If [!NODE_OPTIONS!] == [] ( + set "NODE_OPTIONS=%%i" + ) Else ( + set "NODE_OPTIONS=!NODE_OPTIONS! %%i" + ) + ) +) + +:: Include pre-defined node option +set "NODE_OPTIONS=--no-warnings %NODE_OPTIONS%" -set "NODE_OPTIONS=--no-warnings %NODE_OPTIONS%" && "%NODE%" "%DIR%\src\cli_plugin" %* +TITLE Kibana Server +"%NODE%" "%DIR%\src\cli_plugin" %* :finally diff --git a/src/dev/build/tasks/bin/scripts/kibana.bat b/src/dev/build/tasks/bin/scripts/kibana.bat index 9d8ba359e53af..d3edc92f110a5 100755 --- a/src/dev/build/tasks/bin/scripts/kibana.bat +++ b/src/dev/build/tasks/bin/scripts/kibana.bat @@ -1,6 +1,6 @@ @echo off -SETLOCAL +SETLOCAL ENABLEDELAYEDEXPANSION set SCRIPT_DIR=%~dp0 for %%I in ("%SCRIPT_DIR%..") do set DIR=%%~dpfI @@ -14,7 +14,27 @@ If Not Exist "%NODE%" ( Exit /B 1 ) -set "NODE_OPTIONS=--no-warnings --max-http-header-size=65536 %NODE_OPTIONS%" && "%NODE%" "%DIR%\src\cli" %* +set CONFIG_DIR=%KIBANA_PATH_CONF% +If [%KIBANA_PATH_CONF%] == [] ( + set CONFIG_DIR=%DIR%\config +) + +IF EXIST "%CONFIG_DIR%\node.options" ( + for /F "eol=# tokens=*" %%i in (%CONFIG_DIR%\node.options) do ( + If [!NODE_OPTIONS!] == [] ( + set "NODE_OPTIONS=%%i" + ) Else ( + set "NODE_OPTIONS=!NODE_OPTIONS! %%i" + ) + ) +) + +:: Include pre-defined node option +set "NODE_OPTIONS=--no-warnings --max-http-header-size=65536 %NODE_OPTIONS%" + +:: This should run independently as the last instruction +:: as we need NODE_OPTIONS previously set to expand +"%NODE%" "%DIR%\src\cli" %* :finally diff --git a/src/dev/build/tasks/copy_source_task.js b/src/dev/build/tasks/copy_source_task.js index 32eb7bf8712e3..e34f05bd6cfff 100644 --- a/src/dev/build/tasks/copy_source_task.js +++ b/src/dev/build/tasks/copy_source_task.js @@ -43,6 +43,7 @@ export const CopySourceTask = { 'typings/**', 'webpackShims/**', 'config/kibana.yml', + 'config/node.options', 'tsconfig*.json', '.i18nrc.json', 'kibana.d.ts',