net: upd sockets: incorrect behavior for multiple clients on a single server

[Frauschi]

Describe the bug

In an application, where we have a single UPD socket waiting for incoming data from multiple clients (e.g., a DTLS server with multiple DTLS clients), we ideally want a file descriptor for each “connected” client for easier handling of the connections. This is especially important for DTLS server implementations. For this to work, we create a new socket UDP socket, bind it to the same local IP:Port as the existing listening one, and finally call connect with the IP:Port of the new remote peer. This new additional socket is now more specific than the existing listening socket, resulting in incoming UDP messages from this remote peer forwarded to the new socket, not the general one.
This is the desired behavior, correctly implemented within Linux and used by many DTLS server libraries. See, for example, here: https://tlsalmin.github.io/2018/12/06/DTLS-server.html

In Zephyr, this behavior currently doesn't work correctly. On the one hand, support for SO_REUSEADDR is missing to be able to bind the new socket to the same IP:Port as the existing one. On the other hand, there is a bug in the connect() path for UDP connections, resulting in the remote peer IP address and port not correctly set to be parsed for incoming messages.

To Reproduce

The following code works as intended on Linux, not on Zephyr:

#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#include <errno.h>

#ifdef __ZEPHYR__

#include <zephyr/posix/fcntl.h>
#include <zephyr/net/net_ip.h>
#include <zephyr/net/socket.h>

#else

#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <poll.h>

#define net_addr_ntop inet_ntop
#define net_addr_pton inet_pton

#endif


#ifdef CONFIG_NET_CONFIG_MY_IPV4_ADDR
#define LOCAL_IP CONFIG_NET_CONFIG_MY_IPV4_ADDR
#else
#define LOCAL_IP "192.168.0.10"
#endif

#define LOCAL_PORT 4242


void setblocking(int fd, bool val)
{
	int fl;

	fl = fcntl(fd, F_GETFL, 0);

	if (val) {
		fl &= ~O_NONBLOCK;
	} else 	{
		fl |= O_NONBLOCK;
	}

	fcntl(fd, F_SETFL, fl);
}


int main(void)
{	
	/* Poll variables */
	struct pollfd pollfds[5];
	int num_fds = 0;
	for (int i = 0; i < sizeof(pollfds)/sizeof(pollfds[0]); i++) 
	{
		pollfds[i].fd = -1;
		pollfds[i].events = 0;
		pollfds[i].revents = 0;
	}

	/* UDP server */
	int server_fd = -1;

	/* Clients */
	int client_1_fd = -1;
	int client_2_fd = -1;


	struct sockaddr_in local_addr = {
			.sin_family = AF_INET,
			.sin_port = htons(LOCAL_PORT),
			.sin_addr.s_addr = 0
	};

	net_addr_pton(local_addr.sin_family, LOCAL_IP, &local_addr.sin_addr);
	
	/* Create the UDP server socket */
	server_fd = socket(local_addr.sin_family, SOCK_DGRAM, IPPROTO_UDP);

	/* Make sure we can bind to the address:port */
	int yes = 1;
	setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, (void*)&yes, sizeof(yes));

	/* Bind server socket to its destined IPv4 address */
	bind(server_fd, (struct sockaddr*) &local_addr, sizeof(local_addr)); 

	/* We want non-blocking sockets */
	setblocking(server_fd, false);

	/* Prepare server_fd for poll() */
	pollfds[num_fds].fd = server_fd;
	pollfds[num_fds].events = POLLIN;
	num_fds++;


	while (1) 
	{
		char buf[1500];
		
		struct sockaddr client_addr;
		socklen_t client_addr_len = sizeof(client_addr);

		/* Block and wait for incoming events */
		int ret = poll(pollfds, num_fds, -1);

		if (ret == -1) {
			continue;
		}

		/* Check which fds created an event */
		for (int i = 0; i < num_fds; i++) 
		{
			int fd = pollfds[i].fd;
			short revents = pollfds[i].revents;


			if ((fd == server_fd) && (revents & POLLIN))
			{
				int received = recvfrom(fd, buf, sizeof(buf), 0, &client_addr, &client_addr_len);

				if (received < 0) 
				{
					printf("recvfrom: error %d on fd %d", errno, server_fd);
				} 
				else 
				{
					char client_ip_string[20];
					net_addr_ntop(AF_INET, 
						      &((struct sockaddr_in*) &client_addr)->sin_addr, 
						      client_ip_string, 
						      sizeof(client_ip_string));
					uint16_t client_port = htons(((struct sockaddr_in*) &client_addr)->sin_port);
					buf[received] = '\0';

					if (client_1_fd == -1)
					{
						/* Create a more specific socket to have a direct connection to the new client */
						client_1_fd = socket(client_addr.sa_family, SOCK_DGRAM, IPPROTO_UDP);
						setsockopt(client_1_fd, SOL_SOCKET, SO_REUSEADDR, (void*)&yes, sizeof(yes)); 
						bind(client_1_fd, (struct sockaddr*) &local_addr, sizeof(local_addr));
						setblocking(client_1_fd, false);
						connect(client_1_fd, &client_addr, sizeof(client_addr));

						pollfds[num_fds].fd = client_1_fd;
						pollfds[num_fds].events = POLLIN;
						num_fds++;

						printf("New Connection (fd %d) from %s:%d: %s", 
						       client_1_fd, 
						       client_ip_string, 
						       client_port, 
						       buf);
					}
					else if (client_2_fd == -1) 
					{
						/* Create a more specific socket to have a direct connection to the new client */
						client_2_fd = socket(client_addr.sa_family, SOCK_DGRAM, IPPROTO_UDP);
						setsockopt(client_2_fd, SOL_SOCKET, SO_REUSEADDR, (void*)&yes, sizeof(yes)); 
						bind(client_2_fd, (struct sockaddr*) &local_addr, sizeof(local_addr));
						setblocking(client_2_fd, false);
						connect(client_2_fd, &client_addr, sizeof(client_addr));

						pollfds[num_fds].fd = client_2_fd;
						pollfds[num_fds].events = POLLIN;
						num_fds++;

						printf("New Connection (fd %d) from %s:%d: %s", 
						       client_2_fd, 
						       client_ip_string, 
						       client_port, 
						       buf);
					}
					else
					{
						printf("Data from %s:%d on fd %d: %s", 
						       client_ip_string, 
						       client_port, 
						       fd, 
						       buf);
					}
				}
			}
			else if ((fd == client_1_fd) && (revents & POLLIN))
			{
				int received = recv(fd, buf, sizeof(buf), 0);

				if (received < 0) 
				{
					printf("recv client_1: error %d", errno);
				} 
				else 
				{						
					buf[received] = '\0';
					printf("Data from client_1 (fd %d): %s", client_1_fd, buf);
				}
			}
			else if ((fd == client_2_fd) && (revents & POLLIN))
			{
				int received = recv(fd, buf, sizeof(buf), 0);

				if (received < 0) 
				{
					printf("recv client_2: error %d", errno);
				} 
				else 
				{						
					buf[received] = '\0';
					printf("Data from client_2 (fd %d): %s", client_2_fd, buf);
				}
			}
		}
	}

	return 0;
}

Used with this proj.conf for the Zephyr build:

CONFIG_NETWORKING=y
CONFIG_NET_NATIVE=y
CONFIG_NET_LOG=y
CONFIG_NET_IPV6=n
CONFIG_NET_IPV4=y
CONFIG_NET_UDP=y
CONFIG_NET_TCP=n
CONFIG_NET_SOCKETS=y
CONFIG_NET_L2_ETHERNET=y
CONFIG_NET_SOCKETS_POLL_MAX=5

CONFIG_NET_BUF_DATA_SIZE=1500

CONFIG_INIT_STACKS=y
CONFIG_TEST_RANDOM_GENERATOR=y

CONFIG_NET_CONFIG_SETTINGS=y
CONFIG_NET_MAX_CONN=5
CONFIG_NET_MAX_CONTEXTS=8
CONFIG_POSIX_MAX_FDS=5

CONFIG_NET_CONFIG_MY_IPV4_ADDR="192.168.0.10" 
CONFIG_NET_CONFIG_MY_IPV4_NETMASK="255.255.255" 
CONFIG_NET_CONFIG_MY_IPV4_GW="192.168.0.1" 

CONFIG_NET_SHELL=y

CONFIG_MAIN_STACK_SIZE=4096

CONFIG_DEBUG_OPTIMIZATIONS=y

Expected behavior

The first two UDP clients sending data to the server receive their own socket. When messages are then received from one of these two peers, the specific socket for this peer receives the data and triggers the poll() event. All other peers still trigger the first general socket of the UDP server.