From 3a345f2bae70096c11928447a19d1d0e10abcf15 Mon Sep 17 00:00:00 2001 From: Miss-you Date: Sun, 19 Jan 2020 22:47:16 +0800 Subject: [PATCH 1/6] yousali:Modify bin/apisix to increase the capacity so that the generated nginx.conf file can open the reuseport configuration Judgment logic: 1. If it is a mac system, do not open reuseport 2. If it is a linux system and the kernel version is higher than 3.9.0, then turn on the reuserport switch, and the reuseport option will be added when generating nginx.conf> configuration --- bin/apisix | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/bin/apisix b/bin/apisix index 89b26326258e..8e9e84d0329e 100755 --- a/bin/apisix +++ b/bin/apisix @@ -128,6 +128,17 @@ stream { } server { + {% if enable_reuseport then %} + + {% for _, port in ipairs(stream_proxy.tcp or {}) do %} + listen {*port*} reuseport; + {% end %} + {% for _, port in ipairs(stream_proxy.udp or {}) do %} + listen {*port*} udp reuseport; + {% end %} + + {% else %} {% --if enable_reuseport %} + {% for _, port in ipairs(stream_proxy.tcp or {}) do %} listen {*port*}; {% end %} @@ -135,6 +146,8 @@ stream { listen {*port*} udp; {% end %} + {% end %} + preread_by_lua_block { apisix.stream_preread_phase() } @@ -261,6 +274,22 @@ http { {% end %} server { + {% if enable_reuseport then %} + + listen {* node_listen *} reuseport; + {% if ssl.enable then %} + listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %} reuseport; + {% end %} + + {% if enable_ipv6 then %} + listen [::]:{* node_listen *} reuseport; + {% if ssl.enable then %} + listen [::]:{* node_ssl_listen *} ssl {% if ssl.enable_http2 then %} http2 {% end %} reuseport; + {% end %} + {% end %} {% -- if enable_ipv6 %} + + {% else %} {% -- if enable_reuseport %} + listen {* node_listen *}; {% if ssl.enable then %} listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %}; @@ -273,6 +302,8 @@ http { {% end %} {% end %} {% -- if enable_ipv6 %} + {% end %} {% -- end enable_reuseport %} + ssl_certificate cert/apisix.crt; ssl_certificate_key cert/apisix.key; ssl_session_cache shared:SSL:1m; @@ -437,6 +468,28 @@ local function get_openresty_version() return nil end +local function check_macos_kernel() + local str = "Darwin" + local ret = excute_cmd("uname") + local pos = string.find(ret, str) + if pos == nil then + return false + end + + return true +end + +local function get_linux_kernel_version() + local str = "Linux version " + local ret = excute_cmd("cat /proc/version") + local m = string.match(ret, "[0-9]+\.[0-9]+\.[0-9]+") + if m == nil then + return nil + end + + return m +end + local function split(self, sep) local sep, fields = sep or ":", {} local pattern = string.format("([^%s]+)", sep) @@ -507,6 +560,7 @@ local function init() with_module_status = with_module_status, node_ssl_listen = 9443, -- default value error_log = {level = "warn"}, + enable_reuseport = false, -- default value } if not yaml_conf.apisix then @@ -530,6 +584,27 @@ local function init() sys_conf["worker_processes"] = "auto" end + local check_macos = check_macos_kernel() + if check_macos == true then + enable_reuseport = false + return + end + + --linux try to up enable_reuseport + local kernel_ver = get_linux_kernel_version() + if kernel_ver == nil then + io.stderr:write("can not find kernel version\n") + return + end + + local reuseport_kernel_ver = "3.9.0" + if check_or_version(kernel_ver, reuseport_kernel_ver) then + io.stderr:write("kernel version >=", reuseport_kernel_ver, " current ", kernel_ver, ", up SO_REUSEPORT.\n") + enable_reuseport = true + else + io.stderr:write("kernel version <", reuseport_kernel_ver, " current ", kernel_ver, ", apisix will not open SO_REUSEPORT\n") + end + local conf_render = template.compile(ngx_tpl) local ngxconf = conf_render(sys_conf) From 502830ac5d296c782076fa5d58a72355f002c2f0 Mon Sep 17 00:00:00 2001 From: Miss-you Date: Wed, 22 Jan 2020 12:58:49 +0800 Subject: [PATCH 2/6] yousali::Improve code based on suggested changes Test cases to be added after pull request # 1063 https://github.com/apache/incubator-apisix/pull/1063 --- bin/apisix | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/bin/apisix b/bin/apisix index 8e9e84d0329e..f867dd95a08d 100755 --- a/bin/apisix +++ b/bin/apisix @@ -128,24 +128,11 @@ stream { } server { - {% if enable_reuseport then %} - - {% for _, port in ipairs(stream_proxy.tcp or {}) do %} - listen {*port*} reuseport; - {% end %} - {% for _, port in ipairs(stream_proxy.udp or {}) do %} - listen {*port*} udp reuseport; - {% end %} - - {% else %} {% --if enable_reuseport %} - {% for _, port in ipairs(stream_proxy.tcp or {}) do %} - listen {*port*}; + listen {*port*} {% if enable_reuseport then %} reuseport {% end %}; {% end %} {% for _, port in ipairs(stream_proxy.udp or {}) do %} - listen {*port*} udp; - {% end %} - + listen {*port*} udp {% if enable_reuseport then %} reuseport {% end %}; {% end %} preread_by_lua_block { @@ -274,36 +261,18 @@ http { {% end %} server { - {% if enable_reuseport then %} - - listen {* node_listen *} reuseport; - {% if ssl.enable then %} - listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %} reuseport; - {% end %} - - {% if enable_ipv6 then %} - listen [::]:{* node_listen *} reuseport; - {% if ssl.enable then %} - listen [::]:{* node_ssl_listen *} ssl {% if ssl.enable_http2 then %} http2 {% end %} reuseport; - {% end %} - {% end %} {% -- if enable_ipv6 %} - - {% else %} {% -- if enable_reuseport %} - listen {* node_listen *}; {% if ssl.enable then %} - listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %}; + listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %}; {% end %} {% if enable_ipv6 then %} listen [::]:{* node_listen *}; {% if ssl.enable then %} - listen [::]:{* node_ssl_listen *} ssl {% if ssl.enable_http2 then %} http2 {% end %}; + listen [::]:{* node_ssl_listen *} ssl {% if ssl.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %}; {% end %} {% end %} {% -- if enable_ipv6 %} - {% end %} {% -- end enable_reuseport %} - ssl_certificate cert/apisix.crt; ssl_certificate_key cert/apisix.key; ssl_session_cache shared:SSL:1m; From 9dfcf344a57732dca9723799a94e1c845618b2d4 Mon Sep 17 00:00:00 2001 From: Miss-you Date: Thu, 6 Feb 2020 00:00:36 +0800 Subject: [PATCH 3/6] yousali: Considering that the kernel version of the current production environment basically supports reuseport, I think it is more reasonable to open the REUSEPORT configuration file by default and supporting configuration it. So, it's a better solution is to allow the users to configure the 'enable_reuseport'. --- bin/apisix | 43 ++----------------------------------------- conf/config.yaml | 1 + 2 files changed, 3 insertions(+), 41 deletions(-) diff --git a/bin/apisix b/bin/apisix index f867dd95a08d..4549ad15263b 100755 --- a/bin/apisix +++ b/bin/apisix @@ -437,28 +437,6 @@ local function get_openresty_version() return nil end -local function check_macos_kernel() - local str = "Darwin" - local ret = excute_cmd("uname") - local pos = string.find(ret, str) - if pos == nil then - return false - end - - return true -end - -local function get_linux_kernel_version() - local str = "Linux version " - local ret = excute_cmd("cat /proc/version") - local m = string.match(ret, "[0-9]+\.[0-9]+\.[0-9]+") - if m == nil then - return nil - end - - return m -end - local function split(self, sep) local sep, fields = sep or ":", {} local pattern = string.format("([^%s]+)", sep) @@ -529,7 +507,7 @@ local function init() with_module_status = with_module_status, node_ssl_listen = 9443, -- default value error_log = {level = "warn"}, - enable_reuseport = false, -- default value + enable_reuseport = true, -- default true } if not yaml_conf.apisix then @@ -553,25 +531,8 @@ local function init() sys_conf["worker_processes"] = "auto" end - local check_macos = check_macos_kernel() - if check_macos == true then + if sys_conf["enable_reuseport"] == false then enable_reuseport = false - return - end - - --linux try to up enable_reuseport - local kernel_ver = get_linux_kernel_version() - if kernel_ver == nil then - io.stderr:write("can not find kernel version\n") - return - end - - local reuseport_kernel_ver = "3.9.0" - if check_or_version(kernel_ver, reuseport_kernel_ver) then - io.stderr:write("kernel version >=", reuseport_kernel_ver, " current ", kernel_ver, ", up SO_REUSEPORT.\n") - enable_reuseport = true - else - io.stderr:write("kernel version <", reuseport_kernel_ver, " current ", kernel_ver, ", apisix will not open SO_REUSEPORT\n") end local conf_render = template.compile(ngx_tpl) diff --git a/conf/config.yaml b/conf/config.yaml index 18d1ec3f4113..6d2405e282fb 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -21,6 +21,7 @@ apisix: enable_admin_cors: true # Admin API support CORS response headers. enable_debug: false enable_dev_mode: false # Sets nginx worker_processes to 1 if set to true + enable_reuseport: true # Enable nginx SO_REUSEPORT switch if set to true. enable_ipv6: true config_center: etcd # etcd: use etcd to store the config value # yaml: fetch the config value from local yaml file `/your_path/conf/apisix.yaml` From 7dcdeb4d8abf1629bae79ef66bbb0fbfae9753d4 Mon Sep 17 00:00:00 2001 From: Miss-you Date: Thu, 6 Feb 2020 16:54:58 +0800 Subject: [PATCH 4/6] yousali:set the default value of 'enable_reuseport' in 'config.yaml' --- bin/apisix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bin/apisix b/bin/apisix index 4549ad15263b..76981e88c41a 100755 --- a/bin/apisix +++ b/bin/apisix @@ -261,13 +261,13 @@ http { {% end %} server { - listen {* node_listen *}; + listen {* node_listen *} {% if enable_reuseport then %} reuseport {% end %}; {% if ssl.enable then %} listen {* ssl.listen_port *} ssl {% if ssl.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %}; {% end %} {% if enable_ipv6 then %} - listen [::]:{* node_listen *}; + listen [::]:{* node_listen *} {% if enable_reuseport then %} reuseport {% end %}; {% if ssl.enable then %} listen [::]:{* node_ssl_listen *} ssl {% if ssl.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %}; {% end %} @@ -507,7 +507,6 @@ local function init() with_module_status = with_module_status, node_ssl_listen = 9443, -- default value error_log = {level = "warn"}, - enable_reuseport = true, -- default true } if not yaml_conf.apisix then @@ -531,10 +530,6 @@ local function init() sys_conf["worker_processes"] = "auto" end - if sys_conf["enable_reuseport"] == false then - enable_reuseport = false - end - local conf_render = template.compile(ngx_tpl) local ngxconf = conf_render(sys_conf) From d0193cdc3b910fc7bcd6921769bae54485c6d354 Mon Sep 17 00:00:00 2001 From: Miss-you Date: Thu, 6 Feb 2020 20:10:11 +0800 Subject: [PATCH 5/6] yousali:add test case to test 'nginx.conf' --- .travis/check-nginxconf.sh | 29 +++++++++++++++++++++++++ .travis/linux_apisix_luarocks_runner.sh | 14 ++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 .travis/check-nginxconf.sh diff --git a/.travis/check-nginxconf.sh b/.travis/check-nginxconf.sh new file mode 100644 index 000000000000..20c7625ee5f4 --- /dev/null +++ b/.travis/check-nginxconf.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#check whether the 'reuseport' is in nginx.conf . +matched=`grep -E "listen.*reuseport" -r conf/ | wc -l` +if [ $matched -eq 0 ]; then + echo "failed: nginx.conf file is missing reuseport configuration" + exit 1 +else + echo "passed: nginx.conf file contains reuseport configuration" +fi + +exit 0 diff --git a/.travis/linux_apisix_luarocks_runner.sh b/.travis/linux_apisix_luarocks_runner.sh index 8a494ffdb739..7e3530161bd5 100755 --- a/.travis/linux_apisix_luarocks_runner.sh +++ b/.travis/linux_apisix_luarocks_runner.sh @@ -22,6 +22,16 @@ export_or_prefix() { export OPENRESTY_PREFIX="/usr/local/openresty-debug" } +#check_result shell_name exit_code +check_result() +{ + #echo "input params:$1" + if [ $2 -ne 0 ]; then + echo "shell:$1 exec failed. exit code:$2" + exit $2 + fi +} + do_install() { wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add - sudo apt-get -y update --fix-missing @@ -46,6 +56,8 @@ script() { sudo apisix help sudo apisix init sudo apisix start + sudo bash .travis/check-nginxconf.sh + check_result ".travis/check-nginxconf.sh" $? sudo apisix stop sudo PATH=$PATH ./utils/install-apisix.sh remove @@ -56,6 +68,8 @@ script() { sudo apisix help sudo apisix init sudo apisix start + sudo bash .travis/check-nginxconf.sh + check_result ".travis/check-nginxconf.sh" $? sudo apisix stop sudo luarocks remove rockspec/apisix-master-0.rockspec From b93c44c55da1ca6457e4c59168a9c7ebdcc78a4d Mon Sep 17 00:00:00 2001 From: Miss-you Date: Fri, 7 Feb 2020 12:02:20 +0800 Subject: [PATCH 6/6] yousali:Fix issue with execution in script linux_apisix_luarocks_runner.sh 1.fix issue 'can not find openresty' 2.remove function 'check_result' 3.fix .gitignore --- .gitignore | 3 ++ .travis/check-nginxconf.sh | 2 +- .travis/linux_apisix_luarocks_runner.sh | 37 ++++++++++--------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index e74c4e6a8c5a..880525d93bb8 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ client_body_temp utils/lj-releng .idea/ *.iml + +# .travis +!.travis/*.sh diff --git a/.travis/check-nginxconf.sh b/.travis/check-nginxconf.sh index 20c7625ee5f4..ba8e4b783c8e 100644 --- a/.travis/check-nginxconf.sh +++ b/.travis/check-nginxconf.sh @@ -18,7 +18,7 @@ # #check whether the 'reuseport' is in nginx.conf . -matched=`grep -E "listen.*reuseport" -r conf/ | wc -l` +matched=`grep -E "listen.*reuseport" conf/nginx.conf | wc -l` if [ $matched -eq 0 ]; then echo "failed: nginx.conf file is missing reuseport configuration" exit 1 diff --git a/.travis/linux_apisix_luarocks_runner.sh b/.travis/linux_apisix_luarocks_runner.sh index 7e3530161bd5..9327a477216a 100755 --- a/.travis/linux_apisix_luarocks_runner.sh +++ b/.travis/linux_apisix_luarocks_runner.sh @@ -22,16 +22,6 @@ export_or_prefix() { export OPENRESTY_PREFIX="/usr/local/openresty-debug" } -#check_result shell_name exit_code -check_result() -{ - #echo "input params:$1" - if [ $2 -ne 0 ]; then - echo "shell:$1 exec failed. exit code:$2" - exit $2 - fi -} - do_install() { wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add - sudo apt-get -y update --fix-missing @@ -53,24 +43,27 @@ script() { sudo mkdir -p /usr/local/apisix/deps sudo PATH=$PATH ./utils/install-apisix.sh install - sudo apisix help - sudo apisix init - sudo apisix start - sudo bash .travis/check-nginxconf.sh - check_result ".travis/check-nginxconf.sh" $? - sudo apisix stop + sudo PATH=$PATH apisix help + sudo PATH=$PATH apisix init + sudo PATH=$PATH apisix start + sudo PATH=$PATH apisix stop sudo PATH=$PATH ./utils/install-apisix.sh remove # install APISIX by luarocks sudo luarocks install rockspec/apisix-master-0.rockspec - sudo apisix help - sudo apisix init - sudo apisix start - sudo bash .travis/check-nginxconf.sh - check_result ".travis/check-nginxconf.sh" $? - sudo apisix stop + sudo PATH=$PATH apisix help + sudo PATH=$PATH apisix init + sudo PATH=$PATH apisix start + sudo PATH=$PATH apisix stop + + # make init + # 'make init' operates scripts and related configuration files in the current directory + # The 'apisix' command is a command in the /usr/local/apisix, + # and the configuration file for the operation is in the /usr/local/apisix/conf + sudo PATH=$PATH make init + sudo PATH=$PATH bash .travis/check-nginxconf.sh sudo luarocks remove rockspec/apisix-master-0.rockspec }