Skip to content

Commit 2257644

Browse files
ywkarasywkaras
committed
Add 14 metrics for TCP connections created for tunnels.
Add current and total metrics for TCP connetions towards clients for blind TCP tunnels, and TLS tunnel, forward, and partial blind tunnel SNI-based tunnels. Add current and total metrics for TCP connetions towards servers, for blind TCP tunnels and TLS tunnels. Only partial blind tunnel SNI-based tunnels are counted as TLS tunnels on the outgoing side, because they are only SNI-based tunnels where ATS termitates the TLS connection form the client and originates a new one towards the server.
1 parent a887b33 commit 2257644

23 files changed

+654
-10
lines changed

‎doc/admin-guide/monitoring/statistics/core/http-connection.en.rst‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,10 +164,33 @@ HTTP Connection
164164

165165
Counts the number of times current parent or next parent was detected
166166

167+
.. ts:stat:: global proxy.process.tunnel.total_client_connections_blind_tcp integer
168+
:type: counter
169+
170+
Total number of non-TLS TCP connections for tunnels where the far end is the client
171+
initiated with an HTTP request (such as a CONNECT or WebSocket request).
172+
173+
.. ts:stat:: global proxy.process.tunnel.current_client_connections_blind_tcp integer
174+
:type: counter
175+
176+
Current number of non-TLS TCP connections for tunnels where the far end is the client
177+
initiated with an HTTP request (such as a CONNECT or WebSocket request).
178+
179+
.. ts:stat:: global proxy.process.tunnel.total_server_connections_blind_tcp integer
180+
:type: counter
181+
182+
Total number of TCP connections for tunnels where the far end is the server,
183+
except for those counted by ``proxy.process.tunnel.total_server_connections_tls``
184+
185+
.. ts:stat:: global proxy.process.tunnel.current_server_connections_blind_tcp integer
186+
:type: counter
187+
188+
Current number of TCP connections for tunnels where the far end is the server,
189+
except for those counted by ``proxy.process.tunnel.current_server_connections_tls``
190+
167191
HTTP/2
168192
------
169193

170-
171194
.. ts:stat:: global proxy.process.http2.total_client_connections integer
172195
:type: counter
173196

‎doc/admin-guide/monitoring/statistics/core/ssl.en.rst‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,66 @@ SSL/TLS
238238

239239
A gauge of current active SNI Routing Tunnels.
240240

241+
.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_tunnel integer
242+
:type: counter
243+
244+
Total number of TCP connections for TLS tunnels where the far end is the client
245+
created based on a ``tunnel_route`` key in a table in the :file:`sni.yaml` file.
246+
247+
.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_tunnel integer
248+
:type: counter
249+
250+
Current number of TCP connections for TLS tunnels where the far end is the client
251+
created based on a ``tunnel_route`` key in a table in the :file:`sni.yaml` file.
252+
253+
.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_forward integer
254+
:type: counter
255+
256+
Total number of TCP connections for TLS tunnels where the far end is the client
257+
created based on a ``forward_route`` key in a table in the :file:`sni.yaml` file.
258+
259+
.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_forward integer
260+
:type: counter
261+
262+
Current number of TCP connections for TLS tunnels where the far end is the client
263+
created based on a ``forward_route`` key in a table in the :file:`sni.yaml` file.
264+
265+
.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_partial_blind integer
266+
:type: counter
267+
268+
Total number of TCP connections for TLS tunnels where the far end is the client
269+
created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file.
270+
271+
.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_partial_blind integer
272+
:type: counter
273+
274+
Current number of TCP connections for TLS tunnels where the far end is the client
275+
created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file.
276+
277+
.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_http integer
278+
:type: counter
279+
280+
Total number of TLS connections for tunnels where the far end is the client
281+
initiated with an HTTP request.
282+
283+
.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_http integer
284+
:type: counter
285+
286+
Current number of TLS connections for tunnels where the far end is the client
287+
initiated with an HTTP request.
288+
289+
.. ts:stat:: global proxy.process.tunnel.total_server_connections_tls integer
290+
:type: counter
291+
292+
Total number of TCP connections for TLS tunnels where the far end is the server
293+
created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file.
294+
295+
.. ts:stat:: global proxy.process.tunnel.current_server_connections_tls integer
296+
:type: counter
297+
298+
Current number of TCP connections for TLS tunnels where the far end is the server
299+
created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file.
300+
241301
.. _pre-warming-tls-tunnel-stats:
242302

243303
Pre-warming TLS Tunnel

‎iocore/eventsystem/I_VConnection.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,13 @@ class VConnection : public Continuation
362362
return false;
363363
}
364364

365-
public:
365+
// This function should be called when the VConnection is a tunnel endpoint. By default, a VConnection does not care if it
366+
// is a tunnel endpoint.
367+
virtual void
368+
make_tunnel_endpoint()
369+
{
370+
}
371+
366372
/**
367373
The error code from the last error.
368374

‎iocore/net/Net.cc‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,27 @@ register_net_stats()
9898
};
9999

100100
const std::pair<const char *, Net_Stats> non_persistent[] = {
101-
{"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat },
102-
{"proxy.process.net.connections_currently_open", net_connections_currently_open_stat },
103-
{"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat},
104-
{"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat },
105-
{"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat },
106-
{"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat },
107-
{"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat },
101+
{"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat },
102+
{"proxy.process.net.connections_currently_open", net_connections_currently_open_stat },
103+
{"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat },
104+
{"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat },
105+
{"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat },
106+
{"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat },
107+
{"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat },
108+
{"proxy.process.tunnel.total_client_connections_blind_tcp", tunnel_total_client_connections_blind_tcp_stat },
109+
{"proxy.process.tunnel.current_client_connections_blind_tcp", tunnel_current_client_connections_blind_tcp_stat },
110+
{"proxy.process.tunnel.total_server_connections_blind_tcp", tunnel_total_server_connections_blind_tcp_stat },
111+
{"proxy.process.tunnel.current_server_connections_blind_tcp", tunnel_current_server_connections_blind_tcp_stat },
112+
{"proxy.process.tunnel.total_client_connections_tls_tunnel", tunnel_total_client_connections_tls_tunnel_stat },
113+
{"proxy.process.tunnel.current_client_connections_tls_tunnel", tunnel_current_client_connections_tls_tunnel_stat },
114+
{"proxy.process.tunnel.total_client_connections_tls_forward", tunnel_total_client_connections_tls_forward_stat },
115+
{"proxy.process.tunnel.current_client_connections_tls_forward", tunnel_current_client_connections_tls_forward_stat },
116+
{"proxy.process.tunnel.total_client_connections_tls_partial_blind", tunnel_total_client_connections_tls_partial_blind_stat },
117+
{"proxy.process.tunnel.current_client_connections_tls_partial_blind", tunnel_current_client_connections_tls_partial_blind_stat},
118+
{"proxy.process.tunnel.total_client_connections_tls_http", tunnel_total_client_connections_tls_http_stat },
119+
{"proxy.process.tunnel.current_client_connections_tls_http", tunnel_current_client_connections_tls_http_stat },
120+
{"proxy.process.tunnel.total_server_connections_tls", tunnel_total_server_connections_tls_stat },
121+
{"proxy.process.tunnel.current_server_connections_tls", tunnel_current_server_connections_tls_stat },
108122
};
109123

110124
for (auto &p : persistent) {
@@ -129,6 +143,20 @@ register_net_stats()
129143
NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat);
130144
NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat);
131145
NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat);
146+
NET_CLEAR_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat);
147+
NET_CLEAR_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat);
148+
NET_CLEAR_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat);
149+
NET_CLEAR_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat);
150+
NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_tunnel_stat);
151+
NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_tunnel_stat);
152+
NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_forward_stat);
153+
NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_forward_stat);
154+
NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_partial_blind_stat);
155+
NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_partial_blind_stat);
156+
NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_http_stat);
157+
NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_http_stat);
158+
NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_stat);
159+
NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_stat);
132160

133161
RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT,
134162
static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum);

‎iocore/net/P_Net.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,20 @@ enum Net_Stats {
5757
net_connections_throttled_in_stat,
5858
net_connections_throttled_out_stat,
5959
net_requests_max_throttled_in_stat,
60+
tunnel_total_client_connections_blind_tcp_stat,
61+
tunnel_current_client_connections_blind_tcp_stat,
62+
tunnel_total_server_connections_blind_tcp_stat,
63+
tunnel_current_server_connections_blind_tcp_stat,
64+
tunnel_total_client_connections_tls_tunnel_stat,
65+
tunnel_current_client_connections_tls_tunnel_stat,
66+
tunnel_total_server_connections_tls_stat,
67+
tunnel_current_server_connections_tls_stat,
68+
tunnel_total_client_connections_tls_forward_stat,
69+
tunnel_current_client_connections_tls_forward_stat,
70+
tunnel_total_client_connections_tls_partial_blind_stat,
71+
tunnel_current_client_connections_tls_partial_blind_stat,
72+
tunnel_total_client_connections_tls_http_stat,
73+
tunnel_current_client_connections_tls_http_stat,
6074
Net_Stat_Count
6175
};
6276

‎iocore/net/P_SSLNetVConnection.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,9 @@ class SSLNetVConnection : public UnixNetVConnection,
482482
ssl_error_t _ssl_write_buffer(const void *buf, int64_t nbytes, int64_t &nwritten);
483483
ssl_error_t _ssl_connect();
484484
ssl_error_t _ssl_accept();
485+
486+
void _in_context_tunnel() override;
487+
void _out_context_tunnel() override;
485488
};
486489

487490
typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *);

‎iocore/net/P_UnixNetVConnection.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ struct PollDescriptor;
4444

4545
enum tcp_congestion_control_t { CLIENT_SIDE, SERVER_SIDE };
4646

47+
// WARNING: many or most of the member functions of UnixNetVConnection should only be used when it is instantiated
48+
// directly. They should not be used when UnixNetVConnection is a base class.
4749
class UnixNetVConnection : public NetVConnection, public NetEvent
4850
{
4951
public:
@@ -225,9 +227,25 @@ class UnixNetVConnection : public NetVConnection, public NetEvent
225227

226228
friend void write_to_net_io(NetHandler *, UnixNetVConnection *, EThread *);
227229

230+
// set_context() should be called before calling this member function.
231+
void make_tunnel_endpoint() override;
232+
233+
bool
234+
is_tunnel_endpoint() const
235+
{
236+
return _is_tunnel_endpoint;
237+
}
238+
228239
private:
229240
virtual void *_prepareForMigration();
230241
virtual NetProcessor *_getNetProcessor();
242+
bool _is_tunnel_endpoint{false};
243+
244+
// Called by make_tunnel_endpiont() when the far end of the TCP connection is the active/client end.
245+
virtual void _in_context_tunnel();
246+
247+
// Called by make_tunnel_endpiont() when the far end of the TCP connection is the passive/server end.
248+
virtual void _out_context_tunnel();
231249
};
232250

233251
extern ClassAllocator<UnixNetVConnection> netVCAllocator;

‎iocore/net/SSLNetVConnection.cc‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -996,6 +996,35 @@ SSLNetVConnection::free(EThread *t)
996996
}
997997
con.close();
998998

999+
if (is_tunnel_endpoint()) {
1000+
ink_assert(get_context() != NET_VCONNECTION_UNSET);
1001+
1002+
int c;
1003+
1004+
if (get_context() == NET_VCONNECTION_IN) {
1005+
switch (get_tunnel_type()) {
1006+
case SNIRoutingType::BLIND:
1007+
c = tunnel_current_client_connections_tls_tunnel_stat;
1008+
break;
1009+
case SNIRoutingType::FORWARD:
1010+
c = tunnel_current_client_connections_tls_forward_stat;
1011+
break;
1012+
case SNIRoutingType::PARTIAL_BLIND:
1013+
c = tunnel_current_client_connections_tls_partial_blind_stat;
1014+
break;
1015+
default:
1016+
c = tunnel_current_client_connections_tls_http_stat;
1017+
break;
1018+
}
1019+
} else { // NET_VCONNECTION_OUT
1020+
// Never a tunnel type for out (to server) context.
1021+
ink_assert(get_tunnel_type() == SNIRoutingType::NONE);
1022+
1023+
c = tunnel_current_server_connections_tls_stat;
1024+
}
1025+
NET_DECREMENT_DYN_STAT(c);
1026+
}
1027+
9991028
#if TS_HAS_TLS_EARLY_DATA
10001029
if (_early_data_reader != nullptr) {
10011030
_early_data_reader->dealloc();
@@ -1908,6 +1937,47 @@ SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg)
19081937
return EVENT_DONE;
19091938
}
19101939

1940+
void
1941+
SSLNetVConnection::_in_context_tunnel()
1942+
{
1943+
ink_assert(get_context() == NET_VCONNECTION_IN);
1944+
1945+
int t, c;
1946+
1947+
switch (get_tunnel_type()) {
1948+
case SNIRoutingType::BLIND:
1949+
t = tunnel_total_client_connections_tls_tunnel_stat;
1950+
c = tunnel_current_client_connections_tls_tunnel_stat;
1951+
break;
1952+
case SNIRoutingType::FORWARD:
1953+
t = tunnel_total_client_connections_tls_forward_stat;
1954+
c = tunnel_current_client_connections_tls_forward_stat;
1955+
break;
1956+
case SNIRoutingType::PARTIAL_BLIND:
1957+
t = tunnel_total_client_connections_tls_partial_blind_stat;
1958+
c = tunnel_current_client_connections_tls_partial_blind_stat;
1959+
break;
1960+
default:
1961+
t = tunnel_total_client_connections_tls_http_stat;
1962+
c = tunnel_current_client_connections_tls_http_stat;
1963+
break;
1964+
}
1965+
NET_INCREMENT_DYN_STAT(t);
1966+
NET_INCREMENT_DYN_STAT(c);
1967+
}
1968+
1969+
void
1970+
SSLNetVConnection::_out_context_tunnel()
1971+
{
1972+
ink_assert(get_context() == NET_VCONNECTION_OUT);
1973+
1974+
// Never a tunnel type for out (to server) context.
1975+
ink_assert(get_tunnel_type() == SNIRoutingType::NONE);
1976+
1977+
NET_INCREMENT_DYN_STAT(tunnel_total_server_connections_tls_stat);
1978+
NET_INCREMENT_DYN_STAT(tunnel_current_server_connections_tls_stat);
1979+
}
1980+
19111981
void
19121982
SSLNetVConnection::increment_ssl_version_metric(int version) const
19131983
{

‎iocore/net/UnixNetVConnection.cc‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,6 +1288,8 @@ UnixNetVConnection::clear()
12881288
void
12891289
UnixNetVConnection::free(EThread *t)
12901290
{
1291+
Debug("iocore_net", "Entering UnixNetVConnection::free()");
1292+
12911293
ink_release_assert(t == this_ethread());
12921294

12931295
// close socket fd
@@ -1296,6 +1298,23 @@ UnixNetVConnection::free(EThread *t)
12961298
}
12971299
con.close();
12981300

1301+
if (is_tunnel_endpoint()) {
1302+
Debug("iocore_net", "Freeing UnixNetVConnection that is tunnel endpoint");
1303+
1304+
int c;
1305+
switch (get_context()) {
1306+
case NET_VCONNECTION_IN:
1307+
c = tunnel_current_client_connections_blind_tcp_stat;
1308+
break;
1309+
case NET_VCONNECTION_OUT:
1310+
c = tunnel_current_server_connections_blind_tcp_stat;
1311+
break;
1312+
default:
1313+
ink_release_assert(false);
1314+
}
1315+
NET_DECREMENT_DYN_STAT(c);
1316+
}
1317+
12991318
clear();
13001319
SET_CONTINUATION_HANDLER(this, &UnixNetVConnection::startEvent);
13011320
ink_assert(con.fd == NO_FD);
@@ -1495,3 +1514,38 @@ UnixNetVConnection::set_tcp_congestion_control(int side)
14951514
return -1;
14961515
#endif
14971516
}
1517+
1518+
void
1519+
UnixNetVConnection::make_tunnel_endpoint()
1520+
{
1521+
Debug("iocore_net", "Entering UnixNetVConnection::make_tunnel_endpoint()");
1522+
1523+
ink_assert(!_is_tunnel_endpoint);
1524+
1525+
_is_tunnel_endpoint = true;
1526+
1527+
switch (get_context()) {
1528+
case NET_VCONNECTION_IN:
1529+
_in_context_tunnel();
1530+
break;
1531+
case NET_VCONNECTION_OUT:
1532+
_out_context_tunnel();
1533+
break;
1534+
default:
1535+
ink_release_assert(false);
1536+
}
1537+
}
1538+
1539+
void
1540+
UnixNetVConnection::_in_context_tunnel()
1541+
{
1542+
NET_INCREMENT_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat);
1543+
NET_INCREMENT_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat);
1544+
}
1545+
1546+
void
1547+
UnixNetVConnection::_out_context_tunnel()
1548+
{
1549+
NET_INCREMENT_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat);
1550+
NET_INCREMENT_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat);
1551+
}

0 commit comments

Comments
 (0)