musl integration test missing console from tls connections at process…

… exit (#920)

* (#781) Fix intermittent musl integration test (tls at proc exit).

* (#781) Fix issues noticed during the review process.

src/scopestdlib.c

@@ -112,6 +112,7 @@ extern const char*         scopelibc_gai_strerror(int);
 extern int             scopelibc_gethostname(char *, size_t);
 extern int             scopelibc_getsockname(int, struct sockaddr *, socklen_t *);
 extern int             scopelibc_getsockopt(int, int, int, void *, socklen_t *);
+extern int             scopelibc_setsockopt(int, int, int, const void *, socklen_t);
 extern int             scopelibc_socket(int, int, int);
 extern int             scopelibc_bind(int, const struct sockaddr *, socklen_t);
 extern int             scopelibc_accept(int, struct sockaddr *, socklen_t *);
@@ -123,6 +124,7 @@ extern ssize_t         scopelibc_sendmsg(int, const struct msghdr *, int);
 extern ssize_t         scopelibc_recv(int, void *, size_t, int);
 extern ssize_t         scopelibc_recvmsg(int, struct msghdr *, int);
 extern ssize_t         scopelibc_recvfrom(int, void *, size_t, int, struct sockaddr *, socklen_t *);
+extern int             scopelibc_shutdown(int, int);
 extern int             scopelibc_poll(struct pollfd *, nfds_t, int);
 extern int             scopelibc_select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 extern int             scopelibc_getaddrinfo(const char *, const char *, const struct addrinfo *, struct addrinfo **);
@@ -671,6 +673,11 @@ scope_getsockopt(int sockfd, int level, int optname,  void *restrict optval, soc
     return scopelibc_getsockopt(sockfd, level, optname, optval, optlen);
 }
 
+int
+scope_setsockopt(int sockfd, int level, int optname,  const void *restrict optval, socklen_t optlen) {
+    return scopelibc_setsockopt(sockfd, level, optname, optval, optlen);
+}
+
 int
 scope_socket(int domain, int type, int protocol) {
     return scopelibc_socket(domain, type, protocol);
@@ -726,6 +733,11 @@ scope_recvfrom(int sockfd, void *restrict buf, size_t len, int flags, struct soc
     return scopelibc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen);
 }
 
+int
+scope_shutdown(int sockfd, int how) {
+    return scopelibc_shutdown(sockfd, how);
+}
+
 int
 scope_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
     return scopelibc_poll(fds, nfds, timeout);

src/scopestdlib.h

@@ -158,6 +158,7 @@ const char*        scope_gai_strerror(int);
 int             scope_gethostname(char *, size_t);
 int             scope_getsockname(int, struct sockaddr *, socklen_t *);
 int             scope_getsockopt(int, int, int, void *, socklen_t *);
+int             scope_setsockopt(int, int, int, const void *, socklen_t);
 int             scope_socket(int, int, int);
 int             scope_accept(int, struct sockaddr *, socklen_t *);
 int             scope_bind(int, const struct sockaddr *, socklen_t);
@@ -169,6 +170,7 @@ ssize_t         scope_sendmsg(int, const struct msghdr *, int);
 ssize_t         scope_recv(int, void *, size_t, int);
 ssize_t         scope_recvmsg(int, struct msghdr *, int);
 ssize_t         scope_recvfrom(int, void *, size_t, int, struct sockaddr *, socklen_t *);
+int             scope_shutdown(int, int);
 int             scope_poll(struct pollfd *, nfds_t, int);
 int             scope_select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 int             scope_getaddrinfo(const char *, const char *, const struct addrinfo *, struct addrinfo **);

src/transport.c

@@ -5,6 +5,7 @@
 #include <fcntl.h>
 #include <netdb.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <pthread.h>
 #include <stddef.h>
 #include <stdlib.h>
@@ -291,6 +292,7 @@ shutdownTlsSession(transport_t *trans)
     }
 
     if (trans->net.sock != -1) {
+        scope_shutdown(trans->net.sock, SHUT_RDWR);
         scope_close(trans->net.sock);
         trans->net.sock = -1;
     }
@@ -375,6 +377,12 @@ establishTlsSession(transport_t *trans)
         goto err;
     }
 
+    // This improves the delivery but we're unsure of what the cost is
+    // in terms of network usage.
+    // See https://github.com/criblio/appscope/issues/781
+    //
+    // BIO_set_tcp_ndelay(trans->net.sock, TRUE);
+
     if (trans->net.tls.validateserver) {
         // Just test that we received a server cert
         X509* cert = SSL_get_peer_certificate(trans->net.tls.ssl);
@@ -636,6 +644,26 @@ checkPendingSocketStatus(transport_t *trans)
         DBG("%d %s %s", trans->net.sock, trans->net.host, trans->net.port);
     }
 
+    // Set TCP_QUICKACK
+#if defined(TCP_QUICKACK) && (defined(IPPROTO_TCP) || defined(SOL_TCP))
+    if (trans->type == CFG_TCP) {
+        int opt;
+        int on = TRUE;
+
+#ifdef SOL_TCP
+        opt=SOL_TCP;
+#else
+#ifdef IPPROTO_TCP
+        opt=IPPROTO_TCP;
+#endif
+#endif
+        if (scope_setsockopt(trans->net.sock, opt, TCP_QUICKACK, &on, sizeof(on))) {
+            DBG("%d %s %s", trans->net.sock, trans->net.host, trans->net.port);
+        }
+    }
+#endif
+
+
     // We have a connected socket!  Woot!
     trans->net.connect_attempts = 0;
     trans->net.failure_reason = NO_FAIL;

test/integration/musl/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:latest
 
-RUN apk add bash binutils curl gcc gdb musl-dev openssl openssl-dev
+RUN apk add bash binutils curl gcc gdb musl-dev openssl socat
 
 RUN mkdir -p /opt/fwrite
 COPY ./musl/fwrite.c /opt/fwrite/fwrite.c
@@ -37,9 +37,7 @@ RUN  mkdir /usr/local/scope && \
 COPY musl/scope-test /usr/local/scope/scope-test
 
 
-COPY musl/tcpserver.c /usr/local/scope/tcpserver.c
 RUN (cd /usr/local/scope &&  \
-     gcc -g tcpserver.c -lssl -lcrypto -o tcpserver && \
      openssl req -nodes -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem \
       -subj "/C=US/ST=MN/L=Excelsior/O=Cribl/OU=Engineering/CN=cribl.io")
 

test/integration/musl/scope-test

@@ -128,39 +128,40 @@ fi
 endtest
 
 
-##
-## tls
-##
-## This was written to ensure that #761 stays fixed though it wasn't
-## reliable as written #781 will restore this test
-#starttest tls
 #
-## tcpserver receives over a tls session
-#cd /usr/local/scope
-#./tcpserver -t 9109 > $EVT_FILE &
-#
-## run ldscope with a ton of env vars to send over tls session
-#  SCOPE_EVENT_DEST=tcp://127.0.0.1:9109 \
-#  SCOPE_EVENT_TLS_ENABLE=true \
-#  SCOPE_EVENT_TLS_VALIDATE_SERVER=false \
-#  SCOPE_EVENT_TLS_CA_CERT_PATH=/usr/local/scope/cert.pm \
-#  ldscope ps -ef
-#retval=$?
-#
-#evaltest
-#
-## Verify that ps -ef is successful
-#if [ $retval -ne 0 ]; then
-#    ERR+=1
-#fi
-#
-## Verify that data made it through
-#if ( ! grep '"sourcetype":"console"' $EVT_FILE ); then
-#    ERR+=1
-#fi
-#
-#endtest
+# tls
 #
+starttest tls
+
+# socat receives over a tls session and writes to $EVT_FILE
+cat /usr/local/scope/key.pem /usr/local/scope/cert.pem >> /tmp/appscope.pem
+socat OPENSSL-LISTEN:9109,reuseaddr,cert=/tmp/appscope.pem,cafile=/tmp/appscope.pem,fork,verify=0 $EVT_FILE &
+
+# run ldscope with a ton of env vars to send over tls session
+  SCOPE_EVENT_DEST=tcp://127.0.0.1:9109 \
+  SCOPE_EVENT_TLS_ENABLE=true \
+  SCOPE_EVENT_METRIC=true \
+  SCOPE_EVENT_TLS_CA_CERT_PATH=/tmp/appscope.pem \
+  ldscope ps -ef
+retval=$?
+
+evaltest
+
+# Verify that ps -ef is successful
+if [ $retval -ne 0 ]; then
+    ERR+=1
+fi
+
+# Verify that data made it through
+if ( ! grep '"sourcetype":"console"' $EVT_FILE ); then
+    ERR+=1
+fi
+
+endtest
+
+# cleanup socat, since we're done with it now
+kill `pidof socat`
+
 
 
 if (( $FAILED_TEST_COUNT == 0 )); then