Add OpenSSL 1.1.0 compatibility

Summary:
Build Mysql against Openssl 1.1.0. Most of the changes revolve
around API changes that make OpenSSL structs opaque. Member access
is removed in favor of accessor functions. All changes are version
checked by ifdefs

Test Plan:
Change tools repo to build against 1.1.0 instead of boringssl
Test failures were all due to default cipher differences between the
libraries, or counter differences. Test results are consistent with
those seen before 1eb03e8

Reviewers: anca, avr, mung

Reviewed By: mung

Subscribers: jkedgar, subodh, webscalesql-eng@fb.com, ssl-diffs@fb.com, anca

Differential Revision: https://phabricator.intern.facebook.com/D4743119

Tasks: 16592623

Signature: t1:4743119:1491349557:1d8055b1d4c9ffa2bfd278fa7373149cf6b98df9

Author: abal147

Diff for: fbson/FbsonStream.h

@@ -30,7 +30,7 @@
 #define __STDC_FORMAT_MACROS
 #endif
 
-#include <inttypes.h>
+#include <cinttypes>
 #include <iostream>
 
 namespace fbson {

Diff for: include/violite.h

@@ -148,6 +148,11 @@ int vio_getnameinfo(const struct sockaddr *sa,
 /* Set yaSSL to use same type as MySQL do for socket handles */
 typedef my_socket YASSL_SOCKET_T;
 #define YASSL_SOCKET_T_DEFINED
+#ifdef	__cplusplus
+#include <cinttypes>
+#else
+#include <inttypes.h>
+#endif
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 

Diff for: mysys_ssl/my_aes_openssl.cc

@@ -112,13 +112,13 @@ aes_evp_type(const my_aes_opmode mode)
   }
 }
 
-
 int my_aes_encrypt(const unsigned char *source, uint32 source_length,
                    unsigned char *dest,
                    const unsigned char *key, uint32 key_length,
                    enum my_aes_opmode mode, const unsigned char *iv)
 {
-  EVP_CIPHER_CTX ctx;
+  EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
+  if (!ctx) return MY_AES_BAD_DATA;
   const EVP_CIPHER *cipher= aes_evp_type(mode);
   int u_len, f_len;
   /* The real key to be used for encryption */
@@ -128,23 +128,23 @@ int my_aes_encrypt(const unsigned char *source, uint32 source_length,
   if (!cipher || (EVP_CIPHER_iv_length(cipher) > 0 && !iv))
     return MY_AES_BAD_DATA;
 
-  if (!EVP_EncryptInit(&ctx, cipher, rkey, iv))
+  if (!EVP_EncryptInit(ctx, cipher, rkey, iv))
     goto aes_error;                             /* Error */
-  if (!EVP_CIPHER_CTX_set_padding(&ctx, 1))
+  if (!EVP_CIPHER_CTX_set_padding(ctx, 1))
     goto aes_error;                             /* Error */
-  if (!EVP_EncryptUpdate(&ctx, dest, &u_len, source, source_length))
+  if (!EVP_EncryptUpdate(ctx, dest, &u_len, source, source_length))
     goto aes_error;                             /* Error */
 
-  if (!EVP_EncryptFinal_ex(&ctx, dest + u_len, &f_len))
+  if (!EVP_EncryptFinal_ex(ctx, dest + u_len, &f_len))
     goto aes_error;                             /* Error */
 
-  EVP_CIPHER_CTX_cleanup(&ctx);
+  EVP_CIPHER_CTX_free(ctx);
   return u_len + f_len;
 
 aes_error:
   /* need to explicitly clean up the error if we want to ignore it */
   ERR_clear_error();
-  EVP_CIPHER_CTX_cleanup(&ctx);
+  EVP_CIPHER_CTX_free(ctx);
   return MY_AES_BAD_DATA;
 }
 
@@ -155,7 +155,8 @@ int my_aes_decrypt(const unsigned char *source, uint32 source_length,
                    enum my_aes_opmode mode, const unsigned char *iv)
 {
 
-  EVP_CIPHER_CTX ctx;
+  EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
+  if (!ctx) return MY_AES_BAD_DATA;
   const EVP_CIPHER *cipher= aes_evp_type(mode);
   int u_len, f_len;
 
@@ -166,24 +167,22 @@ int my_aes_decrypt(const unsigned char *source, uint32 source_length,
   if (!cipher || (EVP_CIPHER_iv_length(cipher) > 0 && !iv))
     return MY_AES_BAD_DATA;
 
-  EVP_CIPHER_CTX_init(&ctx);
-
-  if (!EVP_DecryptInit(&ctx, aes_evp_type(mode), rkey, iv))
+  if (!EVP_DecryptInit(ctx, aes_evp_type(mode), rkey, iv))
     goto aes_error;                             /* Error */
-  if (!EVP_CIPHER_CTX_set_padding(&ctx, 1))
+  if (!EVP_CIPHER_CTX_set_padding(ctx, 1))
     goto aes_error;                             /* Error */
-  if (!EVP_DecryptUpdate(&ctx, dest, &u_len, source, source_length))
+  if (!EVP_DecryptUpdate(ctx, dest, &u_len, source, source_length))
     goto aes_error;                             /* Error */
-  if (!EVP_DecryptFinal_ex(&ctx, dest + u_len, &f_len))
+  if (!EVP_DecryptFinal_ex(ctx, dest + u_len, &f_len))
     goto aes_error;                             /* Error */
 
-  EVP_CIPHER_CTX_cleanup(&ctx);
+  EVP_CIPHER_CTX_free(ctx);
   return u_len + f_len;
 
 aes_error:
   /* need to explicitly clean up the error if we want to ignore it */
   ERR_clear_error();
-  EVP_CIPHER_CTX_cleanup(&ctx);
+  EVP_CIPHER_CTX_free(ctx);
   return MY_AES_BAD_DATA;
 }
 

Diff for: sql-common/client.c

@@ -2677,7 +2677,11 @@ static my_bool ssl_check_SAN_IPADD(Vio* vio, GENERAL_NAME* gn_entry)
       DBUG_PRINT("info", ("alternative ip address in cert: %s", unused_ip));
 
       /* Check ipv4 and ipv6 addresses */
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
       char* data= (char*) ASN1_STRING_data(gn_entry->d.ia5);
+#else
+      char* data= (char*) ASN1_STRING_get0_data(gn_entry->d.ia5);
+#endif
       int   length= ASN1_STRING_length(gn_entry->d.ia5);
 
       struct sockaddr* sa= (struct sockaddr*) &vio->remote;
@@ -6898,7 +6902,7 @@ mysql_options4(MYSQL *mysql,enum mysql_option option,
 
      // Increment the reference count
     if (!take_ownership && ssl_session != NULL)
-#ifdef OPENSSL_IS_BORINGSSL
+#if defined(OPENSSL_IS_BORINGSSL) || OPENSSL_VERSION_NUMBER >= 0x10100000L
       SSL_SESSION_up_ref(ssl_session);
 #else
       CRYPTO_add(&ssl_session->references, 1, CRYPTO_LOCK_SSL_SESSION);

Diff for: sql/des_key_file.cc

@@ -13,6 +13,7 @@
    along with this program; if not, write to the Free Software Foundation,
    51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */
 
+#include <cinttypes>
 #include "my_global.h"          // HAVE_*
 #include "sql_priv.h"
 #include "des_key_file.h"       // st_des_keyschedule, st_des_keyblock

Diff for: sql/item_strfunc.cc

@@ -28,6 +28,7 @@
 */
 
 /* May include caustic 3rd-party defs. Use early, so it can override nothing. */
+#include <cinttypes>
 #include "sha2.h"
 #include "my_global.h"                          // HAVE_*
 

Diff for: sql/mysqld.cc

@@ -145,6 +145,11 @@ extern "C" {
 #endif
 #endif
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+// Function removed after OpenSSL 1.1.0
+#define ERR_remove_state(x)
+#endif
+
 using std::min;
 using std::max;
 using std::vector;
@@ -5241,7 +5246,11 @@ bool init_ssl()
 {
 #ifdef HAVE_OPENSSL
 #ifndef HAVE_YASSL
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
   CRYPTO_malloc_init();
+#else
+  OPENSSL_malloc_init();
+#endif // OPENSSL_VERSION_NUMBER
 #endif
   ssl_start();
 #ifndef EMBEDDED_LIBRARY

Diff for: sql/rpl_slave.cc

@@ -74,6 +74,11 @@ bool use_slave_mask = 0;
 MY_BITMAP slave_error_mask;
 char slave_skip_error_names[SHOW_VAR_FUNC_BUFF_SIZE];
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+// Function removed after OpenSSL 1.1.0
+#define ERR_remove_state(x)
+#endif
+
 static unsigned long stop_wait_timeout;
 char* slave_load_tmpdir = 0;
 Master_info *active_mi= 0;

Diff for: sql/sql_class.cc

@@ -5138,26 +5138,27 @@ void THD::get_definer(LEX_USER *definer)
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
 void THD::set_connection_certificate() {
   DBUG_ASSERT(connection_certificate_buf == nullptr);
-  connection_certificate_buf = get_peer_cert_info(false);
+  connection_certificate_buf = get_peer_cert_info(
+    false, &connection_certificate_buf_len);
 }
 
 void THD::reset_connection_certificate() {
   if (connection_certificate_buf) {
-    BUF_MEM_free(connection_certificate_buf);
+    my_free(connection_certificate_buf);
     connection_certificate_buf = nullptr;
+    connection_certificate_buf_len = 0;
   }
 }
 
 const char *THD::connection_certificate() const {
-  return connection_certificate_buf ?
-    connection_certificate_buf->data : nullptr;
+  return connection_certificate_buf;
 }
 
 uint32 THD::connection_certificate_length() const {
-  return connection_certificate_buf ? connection_certificate_buf->length : 0;
+  return connection_certificate_buf ? connection_certificate_buf_len : 0;
 }
 
-BUF_MEM *THD::get_peer_cert_info(bool display)
+char *THD::get_peer_cert_info(bool display, int *cert_len)
 {
   if (!vio_ok() || !net.vio->ssl_arg) {
     return NULL;
@@ -5193,19 +5194,22 @@ BUF_MEM *THD::get_peer_cert_info(bool display)
     return NULL;
   }
 
-  // decouple buffer and close bio object
-  BUF_MEM *bufmem;
-  BIO_get_mem_ptr(bio, &bufmem);
-  (void) BIO_set_close(bio, BIO_NOCLOSE);
-  BIO_free(bio);
-  X509_free(cert);
+  int buflen = BIO_pending(bio);
+  char *cert_buf = (char *)my_malloc(buflen, MYF(MY_WME));
+  *cert_len = BIO_read(bio, cert_buf, buflen);
 
-  if (bufmem->length) {
-    return bufmem;
+  if (*cert_len == -1) {
+    *cert_len = 0;
+    my_free(cert_buf);
+    BIO_free(bio);
+    X509_free(cert);
+    return NULL;
   }
 
-  BUF_MEM_free(bufmem);
-  return NULL;
+  DBUG_ASSERT(*cert_len <= buflen);
+  BIO_free(bio);
+  X509_free(cert);
+  return cert_buf;
 }
 #endif
 

Diff for: sql/sql_class.h

@@ -53,9 +53,6 @@
 #include "sql_data_change.h"
 #include "my_atomic.h"
 
-#include <openssl/ossl_typ.h>
-#include <openssl/pem.h>
-
 #define FLAGSTR(V,F) ((V)&(F)?#F" ":"")
 
 /**
@@ -4543,20 +4540,21 @@ class THD :public MDL_context_owner,
   }
 
 private:
-  BUF_MEM *connection_certificate_buf;
+  char* connection_certificate_buf;
+  int connection_certificate_buf_len;
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
   void reset_connection_certificate();
 public:
   void set_connection_certificate();
   const char *connection_certificate() const;
   uint32 connection_certificate_length() const;
-  // The caller should take ownership of the BUF_MEM pointer.  If display is
+  // The caller should take ownership of the char pointer.  If display is
   // set to true, the buffer will contain the certificate encoded in a human
   // readable format, which can be used for display in information schema.
   // Otherwise, it is encoded in the PEM format.
   //
-  // Free this using the function BUF_MEM_free.
-  BUF_MEM *get_peer_cert_info(bool display);
+  // Free this using the function my_free.
+  char *get_peer_cert_info(bool display, int *cert_len);
 #endif
 
 #ifndef DBUG_OFF

Diff for: sql/sql_show.cc

@@ -2953,10 +2953,11 @@ int fill_schema_authinfo(THD* thd, TABLE_LIST* tables, Item* cond)
     const char* cert = NULL;
     size_t certlen = 0;
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
-    BUF_MEM *bufmem = tmp->get_peer_cert_info(/* display */ true);
+    int tmp_len;
+    char *bufmem = tmp->get_peer_cert_info(/* display */ true, &tmp_len);
     if (bufmem != nullptr) {
-      cert = bufmem->data;
-      certlen = bufmem->length;
+      cert = bufmem;
+      certlen = tmp_len;
     }
 #endif
 
@@ -2967,7 +2968,7 @@ int fill_schema_authinfo(THD* thd, TABLE_LIST* tables, Item* cond)
     }
 
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
-    BUF_MEM_free(bufmem);
+    my_free(bufmem);
 #endif
 
     if (schema_table_store_record(thd, table)) {

Diff for: storage/innobase/include/os0sync.h

@@ -71,18 +71,12 @@ typedef pthread_mutex_t		fast_mutex_t;
 /** Native condition variable */
 typedef pthread_cond_t		os_cond_t;
 
-// The IS_SET macro conflicts with another macro of the same name. Since their
-// usage is mutually exclusive anyway, we define IS_SET (and related macros)
-// only when they are needed. To make use of the macros below in the
-// implementation file, the BIT63_NEEDED variable should be defined.
-#if defined(BIT63_NEEDED)
 #define BIT63 (1ULL << 63)
 #define INC_SIGNAL_COUNT(ev) { ++(ev)->stats; }
 #define SIGNAL_COUNT(ev) (static_cast<ib_int64_t>((ev)->stats & ~BIT63))
 #define SET_IS_SET(ev) { (ev)->stats |= BIT63; }
 #define CLEAR_IS_SET(ev) { (ev)->stats &= ~BIT63; }
 #define IS_SET(ev) (((ev)->stats & BIT63) != 0)
-#endif
 
 #endif
 

Diff for: storage/innobase/os/os0file.cc

@@ -32,7 +32,6 @@ The interface to the operating system file i/o primitives
 Created 10/21/1995 Heikki Tuuri
 *******************************************************/
 
-#define BIT63_NEEDED
 #include "mysqld.h"
 #include "os0file.h"
 

Diff for: storage/innobase/os/os0sync.cc

@@ -24,7 +24,6 @@ synchronization primitives.
 Created 9/6/1995 Heikki Tuuri
 *******************************************************/
 
-#define BIT63_NEEDED
 #include "os0sync.h"
 #include "sync0rw.h"
 #ifdef UNIV_NONINL

Diff for: vio/viossl.c

@@ -483,7 +483,12 @@ static int ssl_init(SSL **out_ssl,
       for (j = 0; j < n; j++)
       {
         SSL_COMP *c = sk_SSL_COMP_value(ssl_comp_methods, j);
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
         DBUG_PRINT("info", ("  %d: %s\n", c->id, c->name));
+#else
+        DBUG_PRINT("info", ("  %d: %s\n",
+              SSL_COMP_get_id(c), SSL_COMP_get0_name(c)));
+#endif
       }
   }
 #endif