Websocket simple proxy protocol implementation

.gitignore

@@ -26,6 +26,11 @@ Module.symvers
 # vscode project settings
 .vscode
 
+# nodejs and npm
+node_modules
+package.json
+package-lock.json
+
 # Python compiler cache files
 *.pyc
 

Makefile

@@ -59,13 +59,14 @@ DBG_CFG ?= 0
 DBG_HTTP_PARSER ?= 0
 DBG_SS ?= 0
 DBG_TLS ?= 0
+DBG_WS ?= 0
 DBG_APM ?= 0
 DBG_HTTP_FRAME ?= 0
 DBG_HTTP_STREAM ?= 0
 DBG_HPACK ?= 0
 TFW_CFLAGS += -DDBG_CFG=$(DBG_CFG) -DDBG_HTTP_PARSER=$(DBG_HTTP_PARSER)
-TFW_CFLAGS += -DDBG_SS=$(DBG_SS) -DDBG_TLS=$(DBG_TLS) -DDBG_APM=$(DBG_APM)
-TFW_CFLAGS += -DDBG_HTTP_FRAME=$(DBG_HTTP_FRAME)
+TFW_CFLAGS += -DDBG_SS=$(DBG_SS) -DDBG_TLS=$(DBG_TLS) -DDBG_WS=$(DBG_WS)
+TFW_CFLAGS += -DDBG_APM=$(DBG_APM) -DDBG_HTTP_FRAME=$(DBG_HTTP_FRAME)
 TFW_CFLAGS += -DDBG_HTTP_STREAM=$(DBG_HTTP_STREAM)
 TFW_CFLAGS += -DDBG_HPACK=$(DBG_HPACK)
 

db/core/htrie.h

@@ -2,7 +2,7 @@
  *		Tempesta DB
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2016 Tempesta Technologies.
+ * Copyright (C) 2015-2022 Tempesta Technologies.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -104,7 +104,7 @@ typedef struct {
 #define TDB_HTRIE_ROOT(h)						\
 	(TdbHtrieNode *)((char *)(h) + TDB_HDR_SZ(h) + sizeof(TdbExt))
 
-/* FIXME we can't store zero bytes by zero key. */
+/* TODO we can't store zero bytes by zero key. */
 static inline int
 tdb_live_fsrec(TdbHdr *dbh, TdbFRec *rec)
 {

db/core/if.c

@@ -3,7 +3,7 @@
  *
  * User-space communication routines.
  *
- * Copyright (C) 2015-2018 Tempesta Technologies, INC.
+ * Copyright (C) 2015-2022 Tempesta Technologies, INC.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -201,7 +201,7 @@ tdb_if_select(struct sk_buff *skb, struct netlink_callback *cb)
 	}
 
 	/*
-	 * FIXME implement select of all records:
+	 * TODO implement select of all records:
 	 * 1. full HTrie iterator is required;
 	 * 2. use many netlink frames to send probably large data set.
 	 */

fw/cache.c

@@ -1347,7 +1347,7 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec,
 	tfw_http_msg_srvhdr_val(h, TFW_HTTP_HDR_ETAG, &h_val);
 
 	/* Update supposed Etag offset to real value. */
-	/* FIXME: #803 */
+	/* TODO: #803 */
 	e_p = TDB_PTR(db->hdr, h_off);
 	if (e_p + TFW_CSTR_HDRLEN > h_trec->data + h_trec->len) {
 		h_trec = tdb_next_rec_chunk(db, h_trec);
@@ -1654,7 +1654,7 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph,
 	}
 
 	/* Update offsets of 304 headers to real values */
-	/* FIXME: #803 */
+	/* TODO: #803 */
 	trec = &ce->trec;
 	for (i = 0; i < ARRAY_SIZE(ce->hdrs_304); ++i) {
 		if (!ce->hdrs_304[i])

fw/cfg.c

@@ -64,7 +64,7 @@
  *  - Improve efficiency: too many memory allocations and data copying.
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2021 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -125,7 +125,7 @@ __alloc_and_copy_literal(const char *src, size_t len, bool keep_bs)
 	}
 
 	/* Copy the string. Eat escaping backslashes if @keep_bs is not set. */
-	/* FIXME: the logic looks like a tiny FSM,
+	/* TODO: the logic looks like a tiny FSM,
 	 *        so perhaps it should be included to the TFSM. */
 	src_end = src + len;
 	src_pos = src;

fw/connection.c

@@ -25,6 +25,7 @@
 #include "log.h"
 #include "sync_socket.h"
 #include "http.h"
+#include "websocket.h"
 
 TfwConnHooks *conn_hooks[TFW_CONN_MAX_PROTOS];
 
@@ -125,14 +126,29 @@ tfw_connection_send(TfwConn *conn, TfwMsg *msg)
 }
 
 int
-tfw_connection_recv(void *cdata, struct sk_buff *skb)
+tfw_connection_recv(TfwConn *conn, struct sk_buff *skb)
 {
-	TfwConn *conn = cdata;
-	TfwFsmData fsm_data = {
-		.skb = skb,
-	};
+	int r = T_OK;
+	struct sk_buff *next;
+
+	if (skb->prev)
+		skb->prev->next = NULL;
+	for (next = skb->next; skb;
+	     skb = next, next = next ? next->next : NULL)
+	{
+		if (likely(r == T_OK || r == T_POSTPONE)) {
+			skb->next = skb->prev = NULL;
+			if (unlikely(TFW_CONN_PROTO(conn) == TFW_FSM_WS
+				     || TFW_CONN_PROTO(conn) == TFW_FSM_WSS))
+				r = tfw_ws_msg_process(conn, skb);
+			else
+				r = tfw_http_msg_process(conn, skb);
+		} else {
+			__kfree_skb(skb);
+		}
+	}
 
-	return tfw_http_msg_process(conn, &fsm_data);
+	return r;
 }
 
 void

fw/connection.h

@@ -51,6 +51,14 @@ enum {
 	/* HTTPS */
 	Conn_HttpsClnt	= Conn_Clnt | TFW_FSM_HTTPS,
 	Conn_HttpsSrv	= Conn_Srv | TFW_FSM_HTTPS,
+
+	/* Websocket plain */
+	Conn_WsClnt	= Conn_HttpClnt | TFW_FSM_WEBSOCKET,
+	Conn_WsSrv	= Conn_HttpSrv | TFW_FSM_WEBSOCKET,
+
+	/* Websocket secure */
+	Conn_WssClnt	= Conn_HttpsClnt | TFW_FSM_WEBSOCKET,
+	Conn_WssSrv	= Conn_HttpsSrv | TFW_FSM_WEBSOCKET,
 };
 
 #define TFW_CONN_TYPE2IDX(t)	TFW_FSM_TYPE(t)
@@ -86,9 +94,11 @@ enum {
  * @timer	- The keep-alive/retry timer for the connection;
  * @stream	- instance for control messages processing;
  * @peer	- TfwClient or TfwServer handler. Hop-by-hop peer;
+ * @pair	- Paired TfwCliConn or TfwSrvConn for websocket connections;
  * @sk		- an appropriate sock handler;
  * @destructor	- called when a connection is destroyed;
  */
+typedef struct tfw_conn_t TfwConn;
 #define TFW_CONN_COMMON					\
 	SsProto			proto;			\
 	TfwGState		state;			\
@@ -97,10 +107,11 @@ enum {
 	struct timer_list	timer;			\
 	TfwStream		stream;			\
 	TfwPeer 		*peer;			\
+	TfwConn			*pair;			\
 	struct sock		*sk;			\
 	void			(*destructor)(void *);
 
-typedef struct TfwConn {
+typedef struct tfw_conn_t {
 	TFW_CONN_COMMON;
 } TfwConn;
 
@@ -199,7 +210,9 @@ enum {
 	/* Connection is in use or at least scheduled to be established. */
 	TFW_CONN_B_ACTIVE,
 	/* Connection is disconnected and stopped. */
-	TFW_CONN_B_STOPPED
+	TFW_CONN_B_STOPPED,
+	/* Mark connection as unavailable to schedulers */
+	TFW_CONN_B_UNSCHED
 };
 
 /**
@@ -297,6 +310,15 @@ tfw_srv_conn_restricted(TfwSrvConn *srv_conn)
 	return test_bit(TFW_CONN_B_RESEND, &srv_conn->flags);
 }
 
+/*
+ * Connection is unavailable to scheduler and may be removed from it
+ */
+static inline bool
+tfw_srv_conn_unscheduled(TfwSrvConn *srv_conn)
+{
+	return test_bit(TFW_CONN_B_UNSCHED, &srv_conn->flags);
+}
+
 /*
  * Tell if a connection has non-idempotent requests.
  */
@@ -499,6 +521,7 @@ tfw_connection_validate_cleanup(TfwConn *conn)
 void tfw_connection_hooks_register(TfwConnHooks *hooks, int type);
 void tfw_connection_hooks_unregister(int type);
 int tfw_connection_send(TfwConn *conn, TfwMsg *msg);
+int tfw_connection_recv(TfwConn *conn, struct sk_buff *skb);
 
 /* Generic helpers, used for both client and server connections. */
 void tfw_connection_init(TfwConn *conn);
@@ -510,6 +533,4 @@ int tfw_connection_close(TfwConn *conn, bool sync);
 void tfw_connection_drop(TfwConn *conn);
 void tfw_connection_release(TfwConn *conn);
 
-int tfw_connection_recv(void *cdata, struct sk_buff *skb);
-
 #endif /* __TFW_CONNECTION_H__ */

fw/gfsm.c

@@ -57,7 +57,7 @@
  * tfw_gfsm_move().
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2018 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2022 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -193,17 +193,6 @@ __gfsm_fsm_exec(TfwGState *st, int fsm_id, TfwFsmData *data)
 	return r;
 }
 
-/**
- * Dispatch connection data to proper FSM by application protocol type.
- */
-int
-tfw_gfsm_dispatch(TfwGState *st, void *obj, TfwFsmData *data)
-{
-	int fsm_id = TFW_FSM_TYPE(((SsProto *)obj)->type);
-
-	return __gfsm_fsm_exec(st, fsm_id, data);
-}
-
 /**
  * Move the FSM with descriptor @st to new the state @state and call all
  * registered hooks for it.
@@ -323,10 +312,6 @@ tfw_gfsm_register_hook(int fsm_id, int prio, int state,
 
 	if (fsm_hooks[fsm_id][shift].fsm_id)
 		return -EBUSY;
-	if (!fsm_htbl[fsm_id]) {
-		T_ERR_NL("gfsm: fsm %d is not registered\n", fsm_id);
-		return -ENOENT;
-	}
 
 	fsm_hooks[fsm_id][shift].st0 = st0;
 	fsm_hooks[fsm_id][shift].fsm_id = hndl_fsm_id;

fw/gfsm.h

@@ -84,6 +84,10 @@ enum {
 	/* Protocols */
 	TFW_FSM_HTTP,
 	TFW_FSM_HTTPS,
+	/* Not really a FSM, used for connection hook registration only */
+	TFW_FSM_WEBSOCKET,
+	TFW_FSM_WS  = TFW_FSM_WEBSOCKET | TFW_FSM_HTTP,
+	TFW_FSM_WSS = TFW_FSM_WEBSOCKET | TFW_FSM_HTTPS,
 
 	/* Security rules enforcement. */
 	TFW_FSM_FRANG_REQ,
@@ -181,11 +185,10 @@ typedef struct {
 			  & ((TFW_GFSM_FSM_MASK << TFW_GFSM_FSM_SHIFT)	\
 			    | TFW_GFSM_STATE_MASK))
 
-typedef struct TfwConn TfwConn;
+typedef struct tfw_conn_t TfwConn;
 typedef int (*tfw_gfsm_handler_t)(TfwConn *conn, TfwFsmData *data);
 
 void tfw_gfsm_state_init(TfwGState *st, void *obj, int st0);
-int tfw_gfsm_dispatch(TfwGState *st, void *obj, TfwFsmData *data);
 int tfw_gfsm_move(TfwGState *st, unsigned short state, TfwFsmData *data);
 
 #ifdef DEBUG