Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(admin) add the worker info to the response of /timers #8999

Merged
merged 3 commits into from
Jun 24, 2022
Merged

fix(admin) add the worker info to the response of /timers #8999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cf331cf
style(runloop) use every timer instead of at timer to rebuild router …
ADD-SP Jun 23, 2022
d7ce17f
fix(admin) add the worker info to the response of `/timers`
ADD-SP Jun 23, 2022
54a01cf
tests(helpers) extend function `wait_timer`
ADD-SP Jun 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 41 additions & 34 deletions autodoc/admin-api/data/admin-api.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,45 +491,52 @@ return {
```

```json
{
"flamegraph": {
"running": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 0\n",
"elapsed_time": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 17\n",
"pending": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 0\n"
},
"sys": {
"running": 0,
"runs": 7,
"pending": 0,
"waiting": 7,
"total": 7
{ "worker": {
"id": 0,
"count": 4,
},
"timers": {
"healthcheck-localhost:8080": {
"name": "healthcheck-localhost:8080",
"meta": {
"name": "@/build/luarocks/share/lua/5.1/resty/counter.lua:71:new()",
"callstack": "@./kong/plugins/prometheus/prometheus.lua:673:init_worker();@/build/luarocks/share/lua/5.1/resty/counter.lua:71:new()"
},
"stats": {
"finish": 2,
"runs": 2,
"elapsed_time": {
"min": 0,
"max": 0,
"avg": 0,
"variance": 0
},
"last_err_msg": ""
}
}
"stats": {
"flamegraph": {
"running": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 0\n",
"elapsed_time": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 17\n",
"pending": "@./kong/init.lua:706:init_worker();@./kong/runloop/handler.lua:1086:before() 0\n"
},
"sys": {
"running": 0,
"runs": 7,
"pending": 0,
"waiting": 7,
"total": 7
},
"timers": {
"healthcheck-localhost:8080": {
"name": "healthcheck-localhost:8080",
"meta": {
"name": "@/build/luarocks/share/lua/5.1/resty/counter.lua:71:new()",
"callstack": "@./kong/plugins/prometheus/prometheus.lua:673:init_worker();@/build/luarocks/share/lua/5.1/resty/counter.lua:71:new()"
},
"stats": {
"finish": 2,
"runs": 2,
"elapsed_time": {
"min": 0,
"max": 0,
"avg": 0,
"variance": 0
},
"last_err_msg": ""
}
}
}
}
}
```

* `flamegraph`: String-encoded timer-related flamegraph data.
* `worker`:
* `id`: The ordinal number of the current Nginx worker processes (starting from number 0).
* `count`: The total number of the Nginx worker processes.
* `stats.flamegraph`: String-encoded timer-related flamegraph data.
You can use [brendangregg/FlameGraph](https://github.com/brendangregg/FlameGraph) to generate flamegraph svgs.
* `sys`: List the number of different type of timers.
* `stats.sys`: List the number of different type of timers.
* `running`: number of running timers.
* `pending`: number of pending timers.
* `waiting`: number of unexpired timers.
Expand Down
12 changes: 11 additions & 1 deletion kong/api/routes/kong.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,17 @@ return {
},
["/timers"] = {
GET = function (self, db, helpers)
return kong.response.exit(200, _G.timerng_stats())
local body = {
worker = {
id = ngx.worker.id(),
count = ngx.worker.count(),
},
stats = kong.timer:stats({
verbose = true,
flamegraph = true,
})
}
return kong.response.exit(200, body)
end
}
}
32 changes: 10 additions & 22 deletions kong/globalpatches.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,43 +72,31 @@ return function(options)
_G.native_timer_at = ngx.timer.at
_G.native_timer_every = ngx.timer.every

local timerng
local _timerng

if options.cli or options.rbusted then
timerng = require("resty.timerng").new({
_timerng = require("resty.timerng").new({
min_threads = 16,
max_threads = 32,
})

timerng:start()
_timerng:start()

else
timerng = require("resty.timerng").new()

-- TODO rename
_G.timerng_start = function (debug)
timerng:start()
timerng:set_debug(debug)
end
_G.timerng = _timerng

else
_timerng = require("resty.timerng").new()
_G.timerng = _timerng
end


_G.ngx.timer.at = function (delay, callback, ...)
return timerng:at(delay, callback, ...)
return _timerng:at(delay, callback, ...)
end

_G.ngx.timer.every = function (interval, callback, ...)
return timerng:every(interval, callback, ...)
return _timerng:every(interval, callback, ...)
end

-- TODO rename
_G.timerng_stats = function ()
return timerng:stats({
verbose = true,
flamegraph = true,
})
end

end


Expand Down
8 changes: 7 additions & 1 deletion kong/init.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,13 @@ function Kong.init_worker()
-- duplicated seeds.
math.randomseed()

_G.timerng_start(kong.configuration.log_level == "debug")

-- setup timerng to _G.kong
kong.timer = _G.timerng
_G.timerng = nil

kong.timer:set_debug(kong.configuration.log_level == "debug")
kong.timer:start()

-- init DB

Expand Down
66 changes: 30 additions & 36 deletions kong/runloop/handler.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,30 +16,30 @@ local PluginsIterator = require "kong.runloop.plugins_iterator"
local instrumentation = require "kong.tracing.instrumentation"


local kong = kong
local type = type
local ipairs = ipairs
local tostring = tostring
local tonumber = tonumber
local setmetatable = setmetatable
local sub = string.sub
local byte = string.byte
local gsub = string.gsub
local find = string.find
local lower = string.lower
local fmt = string.format
local ngx = ngx
local var = ngx.var
local log = ngx.log
local exit = ngx.exit
local exec = ngx.exec
local header = ngx.header
local timer_at = ngx.timer.at
local subsystem = ngx.config.subsystem
local clear_header = ngx.req.clear_header
local http_version = ngx.req.http_version
local unpack = unpack
local escape = require("kong.tools.uri").escape
local kong = kong
local type = type
local ipairs = ipairs
local tostring = tostring
local tonumber = tonumber
local setmetatable = setmetatable
local sub = string.sub
local byte = string.byte
local gsub = string.gsub
local find = string.find
local lower = string.lower
local fmt = string.format
local ngx = ngx
local var = ngx.var
local log = ngx.log
local exit = ngx.exit
local exec = ngx.exec
local header = ngx.header
local timer_at = ngx.timer.at
local subsystem = ngx.config.subsystem
local clear_header = ngx.req.clear_header
local http_version = ngx.req.http_version
local unpack = unpack
local escape = require("kong.tools.uri").escape


local is_http_module = subsystem == "http"
Expand Down Expand Up @@ -1125,14 +1125,11 @@ return {
if not ok then
log(ERR, "could not rebuild router via timer: ", err)
end

local _, err = timer_at(worker_state_update_frequency, rebuild_router_timer)
if err then
log(ERR, "could not schedule timer to rebuild router: ", err)
end
end

local _, err = timer_at(worker_state_update_frequency, rebuild_router_timer)
local _, err = kong.timer:named_every("router-rebuild",
worker_state_update_frequency,
rebuild_router_timer)
if err then
log(ERR, "could not schedule timer to rebuild router: ", err)
end
Expand All @@ -1152,14 +1149,11 @@ return {
if err then
log(ERR, "could not rebuild plugins iterator via timer: ", err)
end

local _, err = timer_at(worker_state_update_frequency, rebuild_plugins_iterator_timer)
if err then
log(ERR, "could not schedule timer to rebuild plugins iterator: ", err)
end
end

local _, err = timer_at(worker_state_update_frequency, rebuild_plugins_iterator_timer)
local _, err = kong.timer:named_every("plugins-iterator-rebuild",
worker_state_update_frequency,
rebuild_plugins_iterator_timer)
if err then
log(ERR, "could not schedule timer to rebuild plugins iterator: ", err)
end
Expand Down
9 changes: 5 additions & 4 deletions spec/01-unit/01-db/08-cache_warmup_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ local cache_warmup = require("kong.cache.warmup")
local helpers = require("spec.helpers")



local function mock_entity(db_data, entity_name, cache_key)
return {
schema = {
Expand Down Expand Up @@ -212,13 +213,13 @@ describe("cache_warmup", function()

cache_warmup._mock_kong(kong)

local runs_old = _G.timerng_stats().sys.runs
local runs_old = _G.timerng:stats().sys.runs

assert.truthy(cache_warmup.execute({"my_entity", "services"}))

-- waiting async DNS cacheing
helpers.wait_until(function ()
local runs = _G.timerng_stats().sys.runs
local runs = _G.timerng:stats().sys.runs
return runs_old < runs
end)

Expand Down Expand Up @@ -271,13 +272,13 @@ describe("cache_warmup", function()

cache_warmup._mock_kong(kong)

local runs_old = _G.timerng_stats().sys.runs
local runs_old = _G.timerng:stats().sys.runs

assert.truthy(cache_warmup.execute({"my_entity", "services"}))

-- waiting async DNS cacheing
helpers.wait_until(function ()
local runs = _G.timerng_stats().sys.runs
local runs = _G.timerng:stats().sys.runs
return runs_old < runs
end)

Expand Down
1 change: 1 addition & 0 deletions spec/01-unit/16-runloop_handler_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ local function setup_it_block()
},

kong = {
timer = _G.timerng,
log = {
err = function() end,
warn = function() end,
Expand Down
2 changes: 1 addition & 1 deletion spec/02-integration/03-db/09-query-semaphore_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe("#postgres Postgres query locks", function()
assert.res_status(204 , res)

-- wait for zero-delay timer
helpers.wait_timer("slow-query", true, true)
helpers.wait_timer("slow-query", true, "any-running")

res = assert(client:send {
method = "GET",
Expand Down
21 changes: 12 additions & 9 deletions spec/02-integration/04-admin_api/20-timers_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,20 @@ local client
local body = assert.res_status(200 , res)
local json = cjson.decode(body)

assert(type(json.flamegraph.running) == "string")
assert(type(json.flamegraph.pending) == "string")
assert(type(json.flamegraph.elapsed_time) == "string")
assert(type(json.worker.id) == "number")
assert(type(json.worker.count) == "number")

assert(type(json.sys.total) == "number")
assert(type(json.sys.runs) == "number")
assert(type(json.sys.running) == "number")
assert(type(json.sys.pending) == "number")
assert(type(json.sys.waiting) == "number")
assert(type(json.stats.flamegraph.running) == "string")
assert(type(json.stats.flamegraph.pending) == "string")
assert(type(json.stats.flamegraph.elapsed_time) == "string")

assert(type(json.timers) == "table")
assert(type(json.stats.sys.total) == "number")
assert(type(json.stats.sys.runs) == "number")
assert(type(json.stats.sys.running) == "number")
assert(type(json.stats.sys.pending) == "number")
assert(type(json.stats.sys.waiting) == "number")

assert(type(json.stats.timers) == "table")

end)

Expand Down
Loading