socket: return EPROTONOSUPPORT on invalid protocol (#16479)

Invalid socket protocol now fails with `EPROTONOSUPPORT` instead of
crashing with an assertion error.

Author: tiran

src/library_sockfs.js

@@ -48,8 +48,8 @@ mergeInto(LibraryManager.library, {
     createSocket: function(family, type, protocol) {
       type &= ~{{{ cDefine('SOCK_CLOEXEC') | cDefine('SOCK_NONBLOCK') }}}; // Some applications may pass it; it makes no sense for a single process.
       var streaming = type == {{{ cDefine('SOCK_STREAM') }}};
-      if (protocol) {
-        assert(streaming == (protocol == {{{ cDefine('IPPROTO_TCP') }}})); // if SOCK_STREAM, must be tcp
+      if (streaming && protocol && protocol != {{{ cDefine('IPPROTO_TCP') }}}) {
+        throw new FS.ErrnoError({{{ cDefine('EPROTONOSUPPORT') }}}); // if SOCK_STREAM, must be tcp or 0.
       }
 
       // create our internal socket structure

tests/sockets/test_create_socket.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2022 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+int main() {
+  int sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+  assert(sockfd >= 0);
+  close(sockfd);
+
+  sockfd = socket(AF_INET, SOCK_STREAM, 0);
+  assert(sockfd >= 0);
+  close(sockfd);
+
+  errno = 0;
+  sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_UDP);
+  assert(sockfd == -1);
+  assert(errno == EPROTONOSUPPORT);
+
+  errno = 0;
+  sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP);
+  assert(sockfd == -1);
+  assert(errno == EPROTONOSUPPORT);
+
+  puts("success");
+
+  return EXIT_SUCCESS;
+}

tests/test_other.py

@@ -8991,6 +8991,9 @@ def test_gethostbyname(self):
   def test_getprotobyname(self):
     self.do_runf(test_file('sockets/test_getprotobyname.c'), 'success')
 
+  def test_create_socket(self):
+    self.do_runf(test_file('sockets/test_create_socket.c'), 'success')
+
   def test_socketpair(self):
     self.do_run(r'''
       #include <sys/socket.h>