Replace GFSM calls with direct calls to TLS and HTTP

Almost literaly follow ak patch from 2eae1da

Replace GFSM calls with direct calls to TLS and HTTP handlers
 on low level networking layers.

GFSM was designed to build graphs of network protocols FSMs (this
design was inspired by FreeBSD netgraph). However, during the years
neither we nor external users have any requirements to introduce
any modules which use GFSM to hook TLS or HTTP entry code. There
are only 2 users of the mechanism for TLS and HTTP for now:
1. TLS -> HTTP protocols handling
2. HTTP limits (the frang module)

This patch replaces GFSM calls with direct calls to
tfw_http_req_process(), tfw_tls_msg_process() and frang_tls_handler()
in following paths:
1. sync sockets -> TLS
2. sync sockets -> HTTP
3. TLS -> HTTP
4. TLS -> Frang

As the result the function tfw_connection_recv() was eliminated.
Now the code is simpler and has lower overhead.

We still might need GFSM for the user-space requests handling (tempesta-tech#77)
and Tempesta Language (tempesta-tech#102).

Contributes to tempesta-tech#755

Based-on-patch-by: Alexander K <ak@tempesta-tech.com>
Signed-off-by: Aleksey Mikhaylov <aym@tempesta-tech.com>

fw/connection.c

@@ -4,7 +4,7 @@
  * Generic connection management.
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2021 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "gfsm.h"
 #include "log.h"
 #include "sync_socket.h"
+#include "http.h"
 
 TfwConnHooks *conn_hooks[TFW_CONN_MAX_PROTOS];
 
@@ -131,7 +132,7 @@ tfw_connection_recv(void *cdata, struct sk_buff *skb)
 		.skb = skb,
 	};
 
-	return tfw_gfsm_dispatch(&conn->state, conn, &fsm_data);
+	return tfw_http_msg_process(conn, &fsm_data);
 }
 
 void

fw/connection.h

@@ -4,7 +4,7 @@
  * Definitions for generic connection management at OSI level 6 (presentation).
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2019 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -100,7 +100,7 @@ enum {
 	struct sock		*sk;			\
 	void			(*destructor)(void *);
 
-typedef struct {
+typedef struct TfwConn {
 	TFW_CONN_COMMON;
 } TfwConn;
 

fw/gfsm.h

@@ -2,7 +2,7 @@
  *		Tempesta FW
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2018 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -88,7 +88,6 @@ enum {
 	/* Security rules enforcement. */
 	TFW_FSM_FRANG_REQ,
 	TFW_FSM_FRANG_RESP,
-	TFW_FSM_FRANG_TLS,
 
 	TFW_FSM_NUM /* Must be <= TFW_GFSM_FSM_N */
 };
@@ -178,11 +177,12 @@ typedef struct {
 	unsigned short	states[TFW_GFSM_FSM_NUM];
 } TfwGState;
 
-#define TFW_GFSM_STATE(s)	((s)->states[(unsigned char)(s)->curr]	\
-				 & ((TFW_GFSM_FSM_MASK << TFW_GFSM_FSM_SHIFT) \
-				    | TFW_GFSM_STATE_MASK))
+#define TFW_GFSM_STATE(s) ((s)->states[(unsigned char)(s)->curr]	\
+			  & ((TFW_GFSM_FSM_MASK << TFW_GFSM_FSM_SHIFT)	\
+			    | TFW_GFSM_STATE_MASK))
 
-typedef int (*tfw_gfsm_handler_t)(void *obj, TfwFsmData *data);
+typedef struct TfwConn TfwConn;
+typedef int (*tfw_gfsm_handler_t)(TfwConn *conn, TfwFsmData *data);
 
 void tfw_gfsm_state_init(TfwGState *st, void *obj, int st0);
 int tfw_gfsm_dispatch(TfwGState *st, void *obj, TfwFsmData *data);

fw/http.c

@@ -117,7 +117,6 @@
 #define RESP_BUF_LEN		128
 
 static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_buf);
-int ghprio; /* GFSM hook priority. */
 
 #define TFW_CFG_BLK_DEF		(TFW_BLK_ERR_REPLY)
 unsigned short tfw_blk_flags = TFW_CFG_BLK_DEF;
@@ -5240,12 +5239,11 @@ tfw_h1_req_process(TfwStream *stream, struct sk_buff *skb)
  * TODO enter the function depending on current GFSM state.
  */
 static int
-tfw_http_req_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
+tfw_http_req_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb)
 {
 	bool block;
 	ss_skb_actor_t *actor;
 	unsigned int parsed;
-	struct sk_buff *skb = data->skb;
 	TfwHttpReq *req;
 	TfwHttpMsg *hmsib;
 	TfwFsmData data_up;
@@ -5276,7 +5274,7 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
 		 skb->len, skb->next, parsed, req->msg.len, req->version, r);
 
 	/*
-	 * We have to keep @data the same to pass it as is to FSMs
+	 * We have to keep @skb the same to pass it as is to FSMs
 	 * registered with lower priorities after us, but we must
 	 * feed the new data version to FSMs registered on our states.
 	 */
@@ -5455,8 +5453,7 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
 	 * the quickest way to obtain target VHost and target backend server
 	 * connection since it allows to avoid expensive tables lookups.
 	 */
-	switch (tfw_http_sess_obtain(req))
-	{
+	switch (tfw_http_sess_obtain(req)) {
 	case TFW_HTTP_SESS_SUCCESS:
 		break;
 
@@ -5851,11 +5848,10 @@ tfw_http_resp_terminate(TfwHttpMsg *hm)
  * TODO enter the function depending on current GFSM state.
  */
 static int
-tfw_http_resp_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
+tfw_http_resp_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb)
 {
 	int r = TFW_BLOCK;
 	unsigned int chunks_unused, parsed;
-	struct sk_buff *skb = data->skb;
 	TfwHttpReq *bad_req;
 	TfwHttpMsg *hmresp, *hmsib;
 	TfwFsmData data_up;
@@ -5890,7 +5886,7 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
 	       skb->len, parsed, hmresp->msg.len, hmresp->version, r);
 
 	/*
-	 * We have to keep @data the same to pass it as is to FSMs
+	 * We have to keep @skb the same to pass it as is to FSMs
 	 * registered with lower priorities after us, but we must
 	 * feed the new data version to FSMs registered on our states.
 	 */
@@ -6075,7 +6071,8 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data)
  * @return status (application logic decision) of the message processing.
  */
 int
-tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream, TfwFsmData *data)
+tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream,
+			     TfwFsmData *data)
 {
 	if (WARN_ON_ONCE(!stream))
 		return -EINVAL;
@@ -6095,8 +6092,8 @@ tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream, TfwFsmData *data)
 	ss_skb_queue_tail(&stream->msg->skb_head, data->skb);
 
 	return (TFW_CONN_TYPE(conn) & Conn_Clnt)
-		? tfw_http_req_process(conn, stream, data)
-		: tfw_http_resp_process(conn, stream, data);
+		? tfw_http_req_process(conn, stream, data->skb)
+		: tfw_http_resp_process(conn, stream, data->skb);
 }
 
 /**
@@ -6108,7 +6105,7 @@ tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream, TfwFsmData *data)
  * returned an error code on. The rest of skbs are freed by us.
  */
 int
-tfw_http_msg_process(void *conn, TfwFsmData *data)
+tfw_http_msg_process(TfwConn *conn, TfwFsmData *data)
 {
 	int r = T_OK;
 	TfwStream *stream = &((TfwConn *)conn)->stream;
@@ -6117,13 +6114,14 @@ tfw_http_msg_process(void *conn, TfwFsmData *data)
 	if (data->skb->prev)
 		data->skb->prev->next = NULL;
 	for (next = data->skb->next; data->skb;
-	     data->skb = next, next = next ? next->next : NULL)
+		data->skb = next, next = next ? next->next : NULL)
 	{
 		if (likely(r == T_OK || r == T_POSTPONE)) {
 			data->skb->next = data->skb->prev = NULL;
 			r = TFW_CONN_H2(conn)
-				? tfw_h2_frame_process(conn, data)
-				: tfw_http_msg_process_generic(conn, stream, data);
+				? tfw_h2_frame_process(conn, data->skb)
+				: tfw_http_msg_process_generic(conn, stream,
+							       data);
 		} else {
 			__kfree_skb(data->skb);
 		}
@@ -6884,22 +6882,11 @@ tfw_http_init(void)
 {
 	int r;
 
-	r = tfw_gfsm_register_fsm(TFW_FSM_HTTP, tfw_http_msg_process);
-	if (r)
+	if ((r = tfw_gfsm_register_fsm(TFW_FSM_HTTP, tfw_http_msg_process)))
 		return r;
 
 	tfw_connection_hooks_register(&http_conn_hooks, TFW_FSM_HTTP);
 
-	ghprio = tfw_gfsm_register_hook(TFW_FSM_TLS,
-					TFW_GFSM_HOOK_PRIORITY_ANY,
-					TFW_TLS_FSM_DATA_READY,
-					TFW_FSM_HTTP, TFW_HTTP_FSM_INIT);
-	if (ghprio < 0) {
-		tfw_connection_hooks_unregister(TFW_FSM_HTTP);
-		tfw_gfsm_unregister_fsm(TFW_FSM_HTTP);
-		return ghprio;
-	}
-
 	tfw_mod_register(&tfw_http_mod);
 
 	return 0;
@@ -6909,7 +6896,6 @@ void
 tfw_http_exit(void)
 {
 	tfw_mod_unregister(&tfw_http_mod);
-	tfw_gfsm_unregister_hook(TFW_FSM_TLS, ghprio, TFW_TLS_FSM_DATA_READY);
 	tfw_connection_hooks_unregister(TFW_FSM_HTTP);
 	tfw_gfsm_unregister_fsm(TFW_FSM_HTTP);
 }

fw/http.h

@@ -668,7 +668,7 @@ tfw_h2_pseudo_index(unsigned short status)
 typedef void (*tfw_http_cache_cb_t)(TfwHttpMsg *);
 
 /* External HTTP functions. */
-int tfw_http_msg_process(void *conn, TfwFsmData *data);
+int tfw_http_msg_process(TfwConn *conn, TfwFsmData *data);
 int tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream,
 				 TfwFsmData *data);
 unsigned long tfw_http_req_key_calc(TfwHttpReq *req);

fw/http_frame.c

@@ -1,7 +1,7 @@
 /**
  *		Tempesta FW
  *
- * Copyright (C) 2019-2021 Tempesta Technologies, Inc.
+ * Copyright (C) 2019-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -1698,14 +1698,14 @@ tfw_h2_context_reinit(TfwH2Ctx *ctx, bool postponed)
 }
 
 int
-tfw_h2_frame_process(void *c, TfwFsmData *data)
+tfw_h2_frame_process(TfwConn *c, struct sk_buff *skb)
 {
 	int r;
 	bool postponed;
 	unsigned int parsed, unused;
 	TfwFsmData data_up = {};
 	TfwH2Ctx *h2 = tfw_h2_context(c);
-	struct sk_buff *nskb = NULL, *skb = data->skb;
+	struct sk_buff *nskb = NULL;
 
 next_msg:
 	postponed = false;

fw/http_frame.h

@@ -200,11 +200,13 @@ typedef struct {
 	unsigned char	data_off;
 } TfwH2Ctx;
 
+typedef struct TfwConn TfwConn;
+
 int tfw_h2_init(void);
 void tfw_h2_cleanup(void);
 int tfw_h2_context_init(TfwH2Ctx *ctx);
 void tfw_h2_context_clear(TfwH2Ctx *ctx);
-int tfw_h2_frame_process(void *c, TfwFsmData *data);
+int tfw_h2_frame_process(TfwConn *c, struct sk_buff *skb);
 void tfw_h2_conn_streams_cleanup(TfwH2Ctx *ctx);
 unsigned int tfw_h2_stream_id(TfwHttpReq *req);
 unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type,

fw/http_limits.c

@@ -855,13 +855,6 @@ enum {
 	TFW_FRANG_RESP_FSM_DONE	= TFW_GFSM_FRANG_RESP_STATE(TFW_GFSM_STATE_LAST)
 };
 
-#define TFW_GFSM_FRANG_TLS_STATE(s)					\
-	((TFW_FSM_FRANG_TLS << TFW_GFSM_FSM_SHIFT) | (s))
-enum {
-	TFW_FRANG_TLS_FSM_INIT	= TFW_GFSM_FRANG_TLS_STATE(0),
-	TFW_FRANG_TLS_FSM_DONE	= TFW_GFSM_FRANG_TLS_STATE(TFW_GFSM_STATE_LAST)
-};
-
 #define __FRANG_FSM_MOVE(st)	T_FSM_MOVE(st, if (r) T_FSM_EXIT(); )
 
 #define __FRANG_FSM_JUMP_EXIT(st)					\
@@ -1245,10 +1238,9 @@ frang_http_req_process(FrangAcc *ra, TfwConn *conn, TfwFsmData *data,
 }
 
 static int
-frang_http_req_handler(void *obj, TfwFsmData *data)
+frang_http_req_handler(TfwConn *conn, TfwFsmData *data)
 {
 	int r;
-	TfwConn *conn = (TfwConn *)obj;
 	FrangAcc *ra = conn->sk->sk_security;
 	TfwVhost *dvh = NULL;
 	TfwHttpReq *req = (TfwHttpReq *)data->req;
@@ -1411,10 +1403,9 @@ frang_resp_fwd_process(TfwHttpResp *resp)
 }
 
 static int
-frang_resp_handler(void *obj, TfwFsmData *data)
+frang_resp_handler(TfwConn *conn, TfwFsmData *data)
 {
 	TfwHttpResp *resp = (TfwHttpResp *)data->resp;
-	TfwConn *conn = (TfwConn *)obj;
 	int r = TFW_PASS;
 
 	switch (TFW_GFSM_STATE(&conn->state)) {
@@ -1450,8 +1441,7 @@ frang_tls_conn_limit(FrangAcc *ra, FrangGlobCfg *conf, int hs_state)
 		ra->history[i].ts = ts;
 	}
 
-	switch (hs_state)
-	{
+	switch (hs_state) {
 	case TTLS_HS_CB_FINISHED_NEW:
 		ra->history[i].tls_sess_new++;
 		break;
@@ -1472,8 +1462,7 @@ frang_tls_conn_limit(FrangAcc *ra, FrangGlobCfg *conf, int hs_state)
 			sum_incomplete += ra->history[i].tls_sess_incomplete;
 		}
 
-	switch (hs_state)
-	{
+	switch (hs_state) {
 	case TTLS_HS_CB_FINISHED_NEW:
 		if (conf->tls_new_conn_rate
 		    && sum_new > conf->tls_new_conn_rate)
@@ -1510,12 +1499,11 @@ frang_tls_conn_limit(FrangAcc *ra, FrangGlobCfg *conf, int hs_state)
 	return TFW_PASS;
 }
 
-static int
-frang_tls_handler(void *obj, TfwFsmData *data)
+int
+frang_tls_handler(TlsCtx *tls, int state)
 {
-	TfwCliConn *conn = (TfwCliConn *)obj;
-	FrangAcc *ra = conn->sk->sk_security;
-	int hs_state = -PTR_ERR(data->req);
+	TfwTlsConn *conn = container_of(tls, TfwTlsConn, tls);
+	FrangAcc *ra = conn->cli_conn.sk->sk_security;
 	TfwVhost *dflt_vh = tfw_vhost_lookup_default();
 	int r;
 
@@ -1524,7 +1512,7 @@ frang_tls_handler(void *obj, TfwFsmData *data)
 
 	spin_lock(&ra->lock);
 
-	r = frang_tls_conn_limit(ra, dflt_vh->frang_gconf, hs_state);
+	r = frang_tls_conn_limit(ra, dflt_vh->frang_gconf, state);
 	if (r == TFW_BLOCK && dflt_vh->frang_gconf->ip_block)
 		tfw_filter_block_ip(&FRANG_ACC2CLI(ra)->addr);
 
@@ -1595,14 +1583,6 @@ static FrangGfsmHook frang_gfsm_hooks[] = {
 		.st0		= TFW_FRANG_RESP_FSM_FWD,
 		.name		= "response_fwd",
 	},
-	{
-		.prio		= -1,
-		.hook_fsm	= TFW_FSM_HTTPS,
-		.hook_state	= TFW_TLS_FSM_HS_DONE,
-		.fsm_id		= TFW_FSM_FRANG_TLS,
-		.st0		= TFW_FRANG_TLS_FSM_INIT,
-		.name		= "tls_hs_done",
-	},
 };
 
 void
@@ -1663,21 +1643,13 @@ tfw_http_limits_init(void)
 		goto err_fsm_resp;
 	}
 
-	r = tfw_gfsm_register_fsm(TFW_FSM_FRANG_TLS, frang_tls_handler);
-	if (r) {
-		T_ERR_NL("frang: can't register frang tls fsm\n");
-		goto err_fsm_tls;
-	}
-
 	r = tfw_http_limits_hooks_register();
 	if (r)
 		goto err_hooks;
 
 	return 0;
 err_hooks:
 	tfw_http_limits_hooks_remove();
-	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_TLS);
-err_fsm_tls:
 	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_RESP);
 err_fsm_resp:
 	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_REQ);
@@ -1693,7 +1665,6 @@ tfw_http_limits_exit(void)
 	T_DBG("frang exit\n");
 
 	tfw_http_limits_hooks_remove();
-	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_TLS);
 	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_RESP);
 	tfw_gfsm_unregister_fsm(TFW_FSM_FRANG_REQ);
 	tfw_classifier_unregister();