tests(healthchecks) add test for success in stream active healthchecks

This serves as a regression test for the problem in
Kong/lua-resty-healthcheck#19

spec/02-integration/05-proxy/10-balancer_spec.lua

@@ -524,6 +524,26 @@ local function wait_for_router_update(bp, old_rv, localhost, proxy_port, admin_p
 end
 
 
+local function tcp_client_requests(nreqs, host, port)
+  local fails, ok1, ok2 = 0, 0, 0
+  for _ = 1, nreqs do
+    local sock = ngx.socket.tcp()
+    assert(sock:connect(host, port))
+    assert(sock:send("hello\n"))
+    local response, err = sock:receive()
+    if err then
+      fails = fails + 1
+      print(err)
+    elseif response:match("^1 ") then
+      ok1 = ok1 + 1
+    elseif response:match("^2 ") then
+      ok2 = ok2 + 1
+    end
+  end
+  return ok1, ok2, fails
+end
+
+
 local function begin_testcase_setup(strategy, bp)
   if strategy == "off" then
     bp.done()
@@ -1294,6 +1314,94 @@ for _, strategy in helpers.each_strategy() do
             end)
           end
 
+          for nchecks = 1, 3 do
+            it("#db perform active health checks -- automatic recovery #stream (nchecks = " .. nchecks .. ")", function()
+
+              local port1 = gen_port()
+              local port2 = gen_port()
+
+              -- setup target servers:
+              -- server2 will only respond for part of the test,
+              -- then server1 will take over.
+              local server1 = helpers.tcp_server(port1, {
+                requests = 1000,
+                prefix = "1 ",
+              })
+              local server2 = helpers.tcp_server(port2, {
+                requests = 1000,
+                prefix = "2 ",
+              })
+              ngx.sleep(0.1)
+
+              -- configure healthchecks
+              begin_testcase_setup(strategy, bp)
+              local upstream_name, upstream_id = add_upstream(bp, {
+                healthchecks = healthchecks_config {
+                  active = {
+                    type = "tcp",
+                    healthy = {
+                      interval = HEALTHCHECK_INTERVAL,
+                      successes = nchecks,
+                    },
+                    unhealthy = {
+                      interval = HEALTHCHECK_INTERVAL,
+                      tcp_failures = nchecks,
+                    },
+                  }
+                }
+              })
+
+              add_target(bp, upstream_id, localhost, port1)
+              add_target(bp, upstream_id, localhost, port2)
+              local _, service_id, route_id = add_api(bp, upstream_name, 500, 500, nil, nil, "tcp")
+              end_testcase_setup(strategy, bp)
+
+              finally(function()
+                helpers.kill_tcp_server(port1)
+                helpers.kill_tcp_server(port2)
+                server1:join()
+                server2:join()
+
+                bp.routes:remove({ id = route_id })
+                bp.services:remove({ id = service_id })
+              end)
+
+              ngx.sleep(0.2)
+
+              -- 1) server1 and server2 take requests
+              local ok1, ok2 = tcp_client_requests(SLOTS * 2, localhost, 9100)
+              assert.same(SLOTS, ok1)
+              assert.same(SLOTS, ok2)
+
+              -- server2 goes unhealthy
+              helpers.kill_tcp_server(port2)
+              server2:join()
+
+              -- Wait until healthchecker detects
+              ngx.sleep(0.2 * nchecks)
+
+              -- 2) server1 takes all requests
+              ok1, ok2 = tcp_client_requests(SLOTS * 2, localhost, 9100)
+              assert.same(SLOTS * 2, ok1)
+              assert.same(0, ok2)
+
+              -- server2 goes healthy again
+              server2 = helpers.tcp_server(port2, {
+                requests = 1000,
+                prefix = "2 ",
+              })
+              ngx.sleep(0.2)
+
+              -- Give time for healthchecker to detect
+              ngx.sleep(0.1 * nchecks)
+
+              -- 3) server1 and server2 take requests again
+              ok1, ok2 = tcp_client_requests(SLOTS * 2, localhost, 9100)
+              assert.same(SLOTS, ok1)
+              assert.same(SLOTS, ok2)
+            end)
+          end
+
           it("perform active health checks -- can detect before any proxy traffic", function()
 
             local nfails = 2
@@ -1593,27 +1701,11 @@ for _, strategy in helpers.each_strategy() do
               requests = server2_oks,
               prefix = "2 ",
             })
-            ngx.sleep(1)
+            ngx.sleep(0.1)
 
             -- server1 and server2 take requests
             -- server1 takes all requests once server2 fails
-            local ok1 = 0
-            local ok2 = 0
-            local fails = 0
-            for _ = 1, SLOTS * 2 do
-              local sock = ngx.socket.tcp()
-              assert(sock:connect(localhost, 9100))
-              assert(sock:send("hello\n"))
-              local response, err = sock:receive()
-              if err then
-                fails = fails + 1
-                print(err)
-              elseif response:match("^1 ") then
-                ok1 = ok1 + 1
-              elseif response:match("^2 ") then
-                ok2 = ok2 + 1
-              end
-            end
+            local ok1, ok2, fails = tcp_client_requests(SLOTS * 2, localhost, 9100)
 
             -- finish up TCP server threads
             server1:join()