FB8-267: porting Add exponential backoff for smart restart

.clang-format

@@ -197,9 +197,6 @@ SpacesInParentheses: false
 SpacesInSquareBrackets: false
 SpaceBeforeSquareBrackets: false
 Standard:        Auto
-StatementMacros:
-  - Q_UNUSED
-  - QT_REQUIRE_VERSION
 TabWidth:        8
 UseCRLF:         false
 UseTab:          Never

.skycastleworkspace.ini

@@ -0,0 +1,3 @@
+[repository]
+name = mysql-5.6
+type = git

CMakeLists.txt

@@ -464,6 +464,13 @@ IF(MY_COMPILER_IS_CLANG OR MY_COMPILER_IS_GNU)
   SET(MY_COMPILER_IS_GNU_OR_CLANG 1)
 ENDIF()
 
+INCLUDE(CTest)
+# Explicitly disabling BUILD_TESTING set by just included CTest, since other
+# code is not ready to handle BUILD_TESTING set. If you try to remove the
+# directive and CMake configuration and reconfiguration finishes successfully,
+# remove the hack.
+SET(BUILD_TESTING OFF)
+
 # Maintainer mode is default on only for debug builds using GCC/G++
 IF(CMAKE_BUILD_TYPE_UPPER STREQUAL "DEBUG" OR WITH_DEBUG)
   IF(MY_COMPILER_IS_GNU)
@@ -570,6 +577,7 @@ INCLUDE(mysql_add_executable)
 INCLUDE(curl)
 INCLUDE(rapidjson)
 INCLUDE(fprofile)
+INCLUDE(prepend_append_cflags_if_supported)
 INCLUDE(gloves)
 
 
@@ -1158,7 +1166,9 @@ ENDIF()
 # Enable debug sync for debug builds.
 FOREACH(LANG C CXX)
   STRING_PREPEND(CMAKE_${LANG}_FLAGS_DEBUG "-DENABLED_DEBUG_SYNC ")
-  STRING_PREPEND(CMAKE_${LANG}_FLAGS_DEBUG "-DSAFE_MUTEX ")
+  IF (NOT DISABLE_SAFE_MUTEX)
+     STRING_PREPEND(CMAKE_${LANG}_FLAGS_DEBUG "-DSAFE_MUTEX ")
+  ENDIF()
 ENDFOREACH()
 
 

azure-pipelines.yml

@@ -216,6 +216,8 @@ jobs:
         -DWITH_BOOST=$(BOOST_DIR)
         -DMYSQL_MAINTAINER_MODE=ON
         -DWITH_SYSTEM_LIBS=ON
+        -DWITH_LZ4=bundled
+        -DWITH_ZSTD=bundled
         -DWITH_MECAB=system
         -DWITH_NUMA=ON
         $SANITIZER_DEFINES

client/check/mysqlcheck_core.cc

@@ -214,7 +214,7 @@ static int use_db(const string &database) {
     return 1;
   if (mysql_get_server_version(sock) >= FIRST_PERFORMANCE_SCHEMA_VERSION &&
       !my_strcasecmp(&my_charset_latin1, database.c_str(),
-                     PERFORMANCE_SCHEMA_DB_NAME))
+                     PERFORMANCE_SCHEMA_DB_NAME_MACRO))
     return 1;
   if (mysql_select_db(sock, database.c_str())) {
     DBError(sock, "when selecting the database");

client/client_priv.h

@@ -160,7 +160,9 @@ enum options_client {
   OPT_SSL_CRLPATH,
   OPT_MYSQLBINLOG_SKIP_GTIDS,
   OPT_MYSQLBINLOG_INCLUDE_GTIDS,
+  OPT_MYSQLBINLOG_INCLUDE_GTIDS_FROM_FILE,
   OPT_MYSQLBINLOG_EXCLUDE_GTIDS,
+  OPT_MYSQLBINLOG_EXCLUDE_GTIDS_FROM_FILE,
   OPT_REMOTE_PROTO,
   OPT_CONFIG_ALL,
   OPT_REWRITE_DB,
@@ -195,6 +197,9 @@ enum options_client {
   OPT_MINIMUM_HLC,
   OPT_CHECKSUM,
   OPT_THREAD_PRIORITY,
+  OPT_COMPRESS_DATA,
+  OPT_COMPRESS_DATA_CHUNK_SIZE,
+  OPT_ORDER_BY_PRIMARY_FORCE_INDEX,
   /* Add new option above this */
   OPT_MAX_CLIENT_OPTION
 };
@@ -217,7 +222,7 @@ enum options_client {
 /**
   Name of the performance schema database.
 */
-#define PERFORMANCE_SCHEMA_DB_NAME "performance_schema"
+#define PERFORMANCE_SCHEMA_DB_NAME_MACRO "performance_schema"
 
 /**
   First mysql version supporting the sys schema.

client/dump/mysqldump_tool_chain_maker_options.cc

@@ -210,7 +210,7 @@ void Mysqldump_tool_chain_maker_options::process_positional_options(
     m_object_filter.m_databases_excluded.push_back(
         std::make_pair("", INFORMATION_SCHEMA_DB_NAME));
     m_object_filter.m_databases_excluded.push_back(
-        std::make_pair("", PERFORMANCE_SCHEMA_DB_NAME));
+        std::make_pair("", PERFORMANCE_SCHEMA_DB_NAME_MACRO));
     m_object_filter.m_databases_excluded.push_back(
         std::make_pair("", "ndbinfo"));
     m_object_filter.m_databases_excluded.push_back(std::make_pair("", "sys"));

client/mysql.cc

@@ -4676,6 +4676,11 @@ static int sql_real_connect(char *host, char *database, char *user,
 static bool init_connection_options(MYSQL *mysql) {
   bool handle_expired = (opt_connect_expired_password || !status.batch);
 
+  // Using the compression_lib connection attribute requires connection
+  // attributes be reset early. Parsing the command arguments could add a new
+  // connection attribute
+  mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, nullptr);
+
   if (opt_init_command)
     mysql_options(mysql, MYSQL_INIT_COMMAND, opt_init_command);
 
@@ -4747,7 +4752,6 @@ static bool init_connection_options(MYSQL *mysql) {
     mysql_options(mysql, MYSQL_ENABLE_CLEARTEXT_PLUGIN,
                   (char *)&opt_enable_cleartext_plugin);
 
-  mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, nullptr);
   mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD, "program_name", "mysql");
   if (current_os_user)
     mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD, "os_user",

client/mysqlbinlog.cc

@@ -807,6 +807,8 @@ enum Exit_status {
   Options that will be used to filter out events.
 */
 static char *opt_include_gtids_str = nullptr, *opt_exclude_gtids_str = nullptr,
+            *opt_include_gtids_from_file_str = nullptr,
+            *opt_exclude_gtids_from_file_str = nullptr,
             *opt_start_gtid_str = nullptr, *opt_find_gtid_str = nullptr,
             *opt_stop_gtid_str = nullptr;
 static char *opt_index_file_str = nullptr;
@@ -2384,11 +2386,21 @@ static struct my_option my_long_options[] = {
      "were provided.",
      &opt_include_gtids_str, &opt_include_gtids_str, nullptr, GET_STR_ALLOC,
      REQUIRED_ARG, 0, 0, 0, nullptr, 0, nullptr},
+    {"include-gtids-from-file", OPT_MYSQLBINLOG_INCLUDE_GTIDS_FROM_FILE,
+     "Print events whose Global Transaction Identifiers "
+     "were provided.",
+     &opt_include_gtids_from_file_str, &opt_include_gtids_from_file_str,
+     nullptr, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, nullptr, 0, nullptr},
     {"exclude-gtids", OPT_MYSQLBINLOG_EXCLUDE_GTIDS,
      "Print all events but those whose Global Transaction "
      "Identifiers were provided.",
      &opt_exclude_gtids_str, &opt_exclude_gtids_str, nullptr, GET_STR_ALLOC,
      REQUIRED_ARG, 0, 0, 0, nullptr, 0, nullptr},
+    {"exclude-gtids-from-file", OPT_MYSQLBINLOG_EXCLUDE_GTIDS_FROM_FILE,
+     "Print all events but those whose Global Transaction "
+     "Identifiers were provided.",
+     &opt_exclude_gtids_from_file_str, &opt_exclude_gtids_from_file_str,
+     nullptr, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, nullptr, 0, nullptr},
     {"print-table-metadata", OPT_PRINT_TABLE_METADATA,
      "Print metadata stored in Table_map_log_event", &opt_print_table_metadata,
      &opt_print_table_metadata, nullptr, GET_BOOL, NO_ARG, 0, 0, 0, nullptr, 0,
@@ -3512,6 +3524,41 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
   return retval;
 }
 
+/* Read gtids file and store the value in the argument 'buffer'.
+ *
+ * @params gtids_file  name of the gtids file
+ * @params buffer[OUT] output buffer to store the gtids string
+ *
+ * @returns FALSE if the reading of the gtids file is successful.
+ *          TRUE  otherwise.
+ */
+static bool read_gtids_file(char *gtids_file, char **buffer) {
+  MY_STAT stat_arg;
+  int fd;
+  /* get the size of the gtids file in bytes */
+  if (!my_stat(gtids_file, &stat_arg, MYF(MY_WME))) return true;
+
+  /* open the gtids file */
+  if ((fd = my_open(gtids_file, O_RDONLY, MYF(MY_WME))) < 0) return true;
+
+  /* allocate buffer */
+  if (!((*buffer) = (char *)my_malloc(
+            PSI_NOT_INSTRUMENTED, (size_t)stat_arg.st_size + 1, MYF(MY_WME)))) {
+    return true;
+  }
+
+  /* read the contents of the file and store in buffer */
+  if (my_read(fd, (uchar *)(*buffer), (size_t)stat_arg.st_size,
+              MYF(MY_FNABP))) {
+    return true;
+  }
+
+  (*buffer)[stat_arg.st_size] = '\0';
+  (void)my_close(fd, MYF(0));
+
+  return false;
+}
+
 /* Post processing of arguments to check for conflicts and other setups */
 static int args_post_process(void) {
   DBUG_TRACE;
@@ -3535,17 +3582,21 @@ static int args_post_process(void) {
       return ERROR_STOP;
     }
 
-    if (opt_include_gtids_str != nullptr) {
-      error("You cannot use --include-gtids and --raw together.");
+    if (opt_include_gtids_str != nullptr ||
+        opt_include_gtids_from_file_str != nullptr) {
+      error("You cannot use --include-gtids%s and --raw together.",
+            opt_include_gtids_from_file_str ? "-from-file" : "");
       return ERROR_STOP;
     }
 
     if (opt_remote_proto == BINLOG_DUMP_NON_GTID &&
-        opt_exclude_gtids_str != nullptr) {
+        (opt_exclude_gtids_str != nullptr ||
+         opt_exclude_gtids_from_file_str != nullptr)) {
       error(
-          "You cannot use both of --exclude-gtids and --raw together "
+          "You cannot use both of --exclude-gtids%s and --raw together "
           "with one of --read-from-remote-server or "
-          "--read-from-remote-master=BINLOG-DUMP-NON-GTID.");
+          "--read-from-remote-master=BINLOG-DUMP-NON-GTID.",
+          opt_exclude_gtids_from_file_str ? "-from-file" : "");
       return ERROR_STOP;
     }
 
@@ -3562,8 +3613,11 @@ static int args_post_process(void) {
     }
   }
 
-  if (opt_start_gtid_str != nullptr && opt_exclude_gtids_str != nullptr) {
-    error("--start-gtid and --exclude-gtids should not be used together");
+  if (opt_start_gtid_str != nullptr &&
+      (opt_exclude_gtids_str != nullptr ||
+       opt_exclude_gtids_from_file_str != nullptr)) {
+    error("--start-gtid and --exclude-gtids%s should not be used together",
+          opt_exclude_gtids_from_file_str ? "-from-file" : "");
     return ERROR_STOP;
   }
 
@@ -3572,7 +3626,10 @@ static int args_post_process(void) {
   if (opt_include_gtids_str != nullptr) {
     if (gtid_set_included->add_gtid_text(opt_include_gtids_str) !=
         RETURN_STATUS_OK) {
-      error("Could not configure --include-gtids '%s'", opt_include_gtids_str);
+      error("Could not configure --include-gtids%s '%s'",
+            opt_include_gtids_from_file_str ? "-from-file" : "",
+            opt_include_gtids_from_file_str ? opt_include_gtids_from_file_str
+                                            : opt_include_gtids_str);
       global_sid_lock->unlock();
       return ERROR_STOP;
     }
@@ -3581,7 +3638,10 @@ static int args_post_process(void) {
   if (opt_exclude_gtids_str != nullptr) {
     if (gtid_set_excluded->add_gtid_text(opt_exclude_gtids_str) !=
         RETURN_STATUS_OK) {
-      error("Could not configure --exclude-gtids '%s'", opt_exclude_gtids_str);
+      error("Could not configure --exclude-gtids%s '%s'",
+            opt_exclude_gtids_from_file_str ? "-from-file" : "",
+            opt_exclude_gtids_from_file_str ? opt_exclude_gtids_from_file_str
+                                            : opt_exclude_gtids_str);
       global_sid_lock->unlock();
       return ERROR_STOP;
     }
@@ -3897,7 +3957,8 @@ int main(int argc, char **argv) {
   parse_args(&argc, &argv);
 
   if (!argc && opt_find_gtid_str == nullptr && opt_start_gtid_str == nullptr &&
-      opt_exclude_gtids_str == nullptr) {
+      (opt_exclude_gtids_str == nullptr ||
+       opt_exclude_gtids_from_file_str != NULL)) {
     usage();
     my_end(my_end_arg);
     return EXIT_FAILURE;
@@ -3915,6 +3976,50 @@ int main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
 
+  if (opt_include_gtids_str != nullptr &&
+      opt_include_gtids_from_file_str != nullptr) {
+    error(
+        "--include-gtids and --include-gtids-from-file should not be used "
+        "together");
+    return EXIT_FAILURE;
+  }
+
+  /* if '--include-gtids-from-file' is specified then read the gtids string
+   * value from the file and store it in 'opt_include_gtids_str'
+   */
+  if (opt_include_gtids_str == nullptr &&
+      opt_include_gtids_from_file_str != nullptr) {
+    if (strlen(opt_include_gtids_from_file_str) > 0 &&
+        read_gtids_file(opt_include_gtids_from_file_str,
+                        &opt_include_gtids_str)) {
+      error("Could not read include-gtids-from-file %s",
+            opt_include_gtids_from_file_str);
+      return EXIT_FAILURE;
+    }
+  }
+
+  if (opt_exclude_gtids_str != nullptr &&
+      opt_exclude_gtids_from_file_str != nullptr) {
+    error(
+        "--exclude-gtids and --exclude-gtids-from-file should not be used "
+        "together");
+    return ERROR_STOP;
+  }
+
+  /* if '--exclude-gtids-from-file' is specified then read the gtids string
+   * value from the file and store it in 'opt_exclude_gtids_str'
+   */
+  if (opt_exclude_gtids_str == nullptr &&
+      opt_exclude_gtids_from_file_str != nullptr) {
+    if (strlen(opt_exclude_gtids_from_file_str) > 0 &&
+        read_gtids_file(opt_exclude_gtids_from_file_str,
+                        &opt_exclude_gtids_str)) {
+      error("Could not read exclude-gtids-from-file %s",
+            opt_exclude_gtids_from_file_str);
+      return EXIT_FAILURE;
+    }
+  }
+
   umask(((~my_umask) & 0666));
   /* Check for argument conflicts and do any post-processing */
   if (args_post_process() == ERROR_STOP) return EXIT_FAILURE;