Fix support for openssl async engine (#6910)

(cherry picked from commit c32fc64)

Author: shinrich

contrib/openssl/README.md

@@ -5,6 +5,6 @@ It should be built as follows.  It must be build against openssl 1.1 or better f
 
 gcc -fPIC -shared -g -o async-test.so -I<path to openssl headers> -L<path to openssl library> -lssl -lcrypto -lpthread async_engine.c
 
-load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine_cnf_file setting.
+load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine.conf_file setting.
 It describes which crypto engines should be loaded and how they should be used.  In the case of our async-test crypto engine it will be used for
 RSA operations

contrib/openssl/async_engine.c

@@ -72,6 +72,15 @@ static int async_rsa_finish(RSA *rsa);
 
 static RSA_METHOD *async_rsa_method = NULL;
 
+EVP_PKEY *async_load_privkey(ENGINE *e, const char *s_key_id, UI_METHOD *ui_method, void *callback_data)
+{
+  printf("Loading key %s\n", s_key_id);
+  FILE *f = fopen(s_key_id, "r");
+  EVP_PKEY *key = PEM_read_PrivateKey(f, NULL, NULL, NULL);
+  fclose(f);
+  return key;
+}
+
 static int bind_async(ENGINE *e)
 {
     /* Setup RSA_METHOD */
@@ -96,7 +105,8 @@ static int bind_async(ENGINE *e)
         || !ENGINE_set_RSA(e, async_rsa_method)
         || !ENGINE_set_destroy_function(e, async_destroy)
         || !ENGINE_set_init_function(e, engine_async_init)
-        || !ENGINE_set_finish_function(e, async_finish)) {
+        || !ENGINE_set_finish_function(e, async_finish)
+        || !ENGINE_set_load_privkey_function(e, async_load_privkey)) {
         fprintf(stderr, "Failed to initialize\n");
         return 0;
     }
@@ -176,14 +186,17 @@ static void async_pause_job(void) {
     OSSL_ASYNC_FD *writefd;
     char buf = DUMMY_CHAR;
 
-    if ((job = ASYNC_get_current_job()) == NULL)
+    if ((job = ASYNC_get_current_job()) == NULL) {
+        printf("No job\n");
         return;
+    }
 
     waitctx = ASYNC_get_wait_ctx(job);
 
     if (ASYNC_WAIT_CTX_get_fd(waitctx, engine_id, &pipefds[0],
                               (void **)&writefd)) {
-        pipefds[1] = *writefd;
+        printf("Existing wait ctx\n");
+        return;
     } else {
         writefd = (OSSL_ASYNC_FD *)OPENSSL_malloc(sizeof(*writefd));
         if (writefd == NULL)
@@ -194,6 +207,8 @@ static void async_pause_job(void) {
         }
         *writefd = pipefds[1];
 
+        printf("New wait ctx %d %d\n", pipefds[0], pipefds[1]);
+
         if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0],
                                        writefd, wait_cleanup)) {
             wait_cleanup(waitctx, engine_id, pipefds[0], writefd);
@@ -213,35 +228,39 @@ void *
 delay_method(void *arg)
 {
   int signal_fd = (intptr_t)arg;
-  sleep(5);
-  uint64_t buf = 1;
+  sleep(2);
+  char buf = DUMMY_CHAR;
   write(signal_fd, &buf, sizeof(buf));
+  printf("Send signal to %d\n", signal_fd);
+  return NULL;
 }
 
 
 void
 spawn_delay_thread()
 {
   pthread_t thread_id;
-  OSSL_ASYNC_FD signal_fd;
-  OSSL_ASYNC_FD pipefds[2] = {0, 0};
   ASYNC_JOB *job;
-  if ((job = ASYNC_get_current_job()) == NULL)
+  if ((job = ASYNC_get_current_job()) == NULL) {
+      printf("Spawn no job\n");
       return;
+  }
 
   ASYNC_WAIT_CTX *waitctx = ASYNC_get_wait_ctx(job);
 
   size_t numfds;
-  if (ASYNC_WAIT_CTX_get_all_fds(waitctx, &signal_fd, &numfds) && numfds > 0) {
+  if (ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds) && numfds > 0) {
+    printf("Spawn, wait_ctx exists.  Go away, something else is using this job\n");
   } else {
+    OSSL_ASYNC_FD signal_fd;
     OSSL_ASYNC_FD pipefds[2] = {0,0};
     OSSL_ASYNC_FD *writefd = OPENSSL_malloc(sizeof(*writefd));
     pipe(pipefds);
     signal_fd = *writefd = pipefds[1];
     ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0], writefd, wait_cleanup);
+    printf("Spawn, create wait_ctx %d %d\n", pipefds[0], pipefds[1]);
+    pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd));
   }
-
-  pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd));
 }
 
 
@@ -264,7 +283,7 @@ static int async_pub_dec(int flen, const unsigned char *from,
 static int async_rsa_priv_enc(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
-    //printf("async_priv_enc\n");
+    printf("async_priv_enc\n");
     spawn_delay_thread();
     async_pause_job();
     return RSA_meth_get_priv_enc(RSA_PKCS1_OpenSSL())
@@ -274,7 +293,7 @@ static int async_rsa_priv_enc(int flen, const unsigned char *from,
 static int async_rsa_priv_dec(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
-    //printf("async_priv_dec\n");
+    printf("async_priv_dec\n");
     spawn_delay_thread();
     async_pause_job();
     return RSA_meth_get_priv_dec(RSA_PKCS1_OpenSSL())

iocore/net/P_UnixNet.h

@@ -97,18 +97,21 @@ struct EventIO {
     UnixUDPConnection *uc;
   } data; ///< a kind of continuation
 
-  int start(EventLoop l, DNSConnection *vc, int events);
-  int start(EventLoop l, NetAccept *vc, int events);
-  int start(EventLoop l, NetEvent *ne, int events);
-  int start(EventLoop l, UnixUDPConnection *vc, int events);
-  /** Setup a continuation to be called when a file descriptor is available for read or write.
+  /** The start methods all logically Setup a class to be called
+     when a file descriptor is available for read or write.
+     The type of the classes vary.  Generally the file descriptor
+     is pulled from the class, but there is one option that lets
+     the file descriptor be expressed directly.
      @param l the event loop
-     @param fd file descriptor (or port)
-     @param c the continuation to call
      @param events a mask of flags (for details `man epoll_ctl`)
      @return int the number of events created, -1 is error
    */
-  int start(EventLoop l, int fd, Continuation *c, int events);
+  int start(EventLoop l, DNSConnection *vc, int events);
+  int start(EventLoop l, NetAccept *vc, int events);
+  int start(EventLoop l, NetEvent *ne, int events);
+  int start(EventLoop l, UnixUDPConnection *vc, int events);
+  int start(EventLoop l, int fd, NetEvent *ne, int events);
+  int start_common(EventLoop l, int fd, int events);
 
   /** Alter the events that will trigger the continuation, for level triggered I/O.
      @param events add with positive mask(+EVENTIO_READ), or remove with negative mask (-EVENTIO_READ)
@@ -577,27 +580,33 @@ write_disable(NetHandler *nh, NetEvent *ne)
 TS_INLINE int
 EventIO::start(EventLoop l, DNSConnection *vc, int events)
 {
-  type = EVENTIO_DNS_CONNECTION;
-  return start(l, vc->fd, (Continuation *)vc, events);
+  type        = EVENTIO_DNS_CONNECTION;
+  data.dnscon = vc;
+  return start_common(l, vc->fd, events);
 }
 TS_INLINE int
 EventIO::start(EventLoop l, NetAccept *vc, int events)
 {
-  type = EVENTIO_NETACCEPT;
-  return start(l, vc->server.fd, (Continuation *)vc, events);
+  type    = EVENTIO_NETACCEPT;
+  data.na = vc;
+  return start_common(l, vc->server.fd, events);
 }
 TS_INLINE int
 EventIO::start(EventLoop l, NetEvent *ne, int events)
 {
-  type = EVENTIO_READWRITE_VC;
-  return start(l, ne->get_fd(), (Continuation *)ne, events);
+  type    = EVENTIO_READWRITE_VC;
+  data.ne = ne;
+  return start_common(l, ne->get_fd(), events);
 }
+
 TS_INLINE int
 EventIO::start(EventLoop l, UnixUDPConnection *vc, int events)
 {
-  type = EVENTIO_UDP_CONNECTION;
-  return start(l, vc->fd, (Continuation *)vc, events);
+  type    = EVENTIO_UDP_CONNECTION;
+  data.uc = vc;
+  return start_common(l, vc->fd, events);
 }
+
 TS_INLINE int
 EventIO::close()
 {
@@ -624,13 +633,19 @@ EventIO::close()
 }
 
 TS_INLINE int
-EventIO::start(EventLoop l, int afd, Continuation *c, int e)
+EventIO::start(EventLoop l, int afd, NetEvent *ne, int e)
+{
+  data.ne = ne;
+  return start_common(l, afd, e);
+}
+
+TS_INLINE int
+EventIO::start_common(EventLoop l, int afd, int e)
 {
   if (!this->syscall) {
     return 0;
   }
 
-  data.c     = c;
   fd         = afd;
   event_loop = l;
 #if TS_USE_EPOLL

iocore/net/SSLNetVConnection.cc

@@ -1233,18 +1233,22 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err)
 #if TS_USE_TLS_ASYNC
   if (ssl_error == SSL_ERROR_WANT_ASYNC) {
     size_t numfds;
-    OSSL_ASYNC_FD waitfd;
+    OSSL_ASYNC_FD *waitfds;
     // Set up the epoll entry for the signalling
-    if (SSL_get_all_async_fds(ssl, &waitfd, &numfds) && numfds > 0) {
-      // Temporarily disable regular net
-      read_disable(nh, this);
-      this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol
-      // Have to have the read NetState enabled because we are using it for the signal vc
-      read.enabled = true;
-      write_disable(nh, this);
-      PollDescriptor *pd = get_PollDescriptor(this_ethread());
-      this->ep.start(pd, waitfd, this, EVENTIO_READ);
-      this->ep.type = EVENTIO_READWRITE_VC;
+    if (SSL_get_all_async_fds(ssl, nullptr, &numfds) && numfds > 0) {
+      // Allocate space for the waitfd on the stack, should only be one most all of the time
+      waitfds = reinterpret_cast<OSSL_ASYNC_FD *>(alloca(sizeof(OSSL_ASYNC_FD) * numfds));
+      if (SSL_get_all_async_fds(ssl, waitfds, &numfds) && numfds > 0) {
+        // Temporarily disable regular net
+        this->read.triggered  = false;
+        this->write.triggered = false;
+        this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol
+        // Have to have the read NetState enabled because we are using it for the signal vc
+        read.enabled       = true;
+        PollDescriptor *pd = get_PollDescriptor(this_ethread());
+        this->ep.start(pd, waitfds[0], static_cast<NetEvent *>(this), EVENTIO_READ);
+        this->ep.type = EVENTIO_READWRITE_VC;
+      }
     }
   } else if (SSLConfigParams::async_handshake_enabled) {
     // Clean up the epoll entry for signalling

iocore/net/SSLUtils.cc

@@ -921,7 +921,12 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str
 #ifndef OPENSSL_IS_BORINGSSL
   ENGINE *e = ENGINE_get_default_RSA();
   if (e != nullptr) {
-    const char *argkey = (keyPath == nullptr || keyPath[0] == '\0') ? completeServerCertPath.c_str() : keyPath;
+    ats_scoped_str argkey;
+    if (keyPath == nullptr || keyPath[0] == '\0') {
+      argkey = completeServerCertPath.c_str();
+    } else {
+      argkey = Layout::get()->relative_to(params->serverKeyPathOnly, keyPath);
+    }
     if (!SSL_CTX_use_PrivateKey(ctx, ENGINE_load_private_key(e, argkey, nullptr, nullptr))) {
       SSLError("failed to load server private key from engine");
     }

tests/gold_tests/tls/tls_engine.test.py

@@ -0,0 +1,98 @@
+'''
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+import os
+
+
+# Someday should add client cert to origin to exercise the
+# engine interface on the other side
+
+Test.Summary = '''
+Test tls via the async interface with the sample async_engine
+'''
+
+Test.SkipUnless(Condition.HasOpenSSLVersion('1.1.1'))
+
+# Define default ATS
+ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True)
+server = Test.MakeOriginServer("server")
+
+# Compile with tsxs.  That should bring in the consisten versions of openssl
+ts.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, '../../contrib/openssl', 'async_engine.c'), Test.RunDirectory)
+ts.Setup.RunCommand("tsxs -o async_engine.so async_engine.c")
+
+# Add info the origin server responses
+server.addResponse("sessionlog.json",
+                   {"headers": "GET / HTTP/1.1\r\nuuid: basic\r\n\r\n", "timestamp": "1469733493.993", "body": ""},
+                   {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"})
+
+# add ssl materials like key, certificates for the server
+ts.addSSLfile("ssl/server.pem")
+ts.addSSLfile("ssl/server.key")
+
+ts.Disk.remap_config.AddLine(
+    'map / http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+
+ts.Disk.ssl_multicert_config.AddLine(
+    'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key'
+)
+ts.Disk.records_config.update({
+    'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir),
+    'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir),
+    'proxy.config.ssl.client.verify.server':  0,
+    'proxy.config.exec_thread.autoconfig.scale': 1.0,
+    'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2',
+    'proxy.config.ssl.engine.conf_file': '{0}/ts/config/load_engine.cnf'.format(Test.RunDirectory),
+    'proxy.config.ssl.async.handshake.enabled': 1,
+    'proxy.config.diags.debug.enabled': 0,
+    'proxy.config.diags.debug.tags': 'ssl'
+})
+
+ts.Disk.MakeConfigFile('load_engine.cnf').AddLines([
+    'openssl_conf = openssl_init',
+    '',
+    '[openssl_init]',
+    '',
+    'engines = engine_section',
+    '',
+    '[engine_section]',
+    '',
+    'async = async_section',
+    '',
+    '[async_section]',
+    '',
+    'dynamic_path = {0}/async_engine.so'.format(Test.RunDirectory),
+    '',
+    'engine_id = async-test',
+    '',
+    'default_algorithms = RSA',
+    '',
+    'init = 1'])
+
+# Make a basic request.  Hopefully it goes through
+tr = Test.AddTestRun("Run-Test")
+tr.Processes.Default.Command = "curl -k -v -H uuid:basic -H host:example.com  https://127.0.0.1:{0}/".format(ts.Variables.ssl_port)
+tr.ReturnCode = 0
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port))
+tr.Processes.Default.Streams.All = Testers.ContainsExpression("HTTP/(2|1\.1) 200", "Request succeeds")
+tr.StillRunningAfter = server
+
+ts.Streams.All += Testers.ContainsExpression("Send signal to ", "The Async engine triggers")