Merge branch 'gvmd-9.0' into nvti-tags-fix-9.0

greenbone · Dec 23, 2019 · a83f777 · a83f777
2 parents a33ca03 + dd35144
commit a83f777
Show file tree

Hide file tree

Showing 3 changed files with 92 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Speed up the SCAP sync [#875](https://github.com/greenbone/gvmd/pull/875) [#877](https://github.com/greenbone/gvmd/pull/877) [#879](https://github.com/greenbone/gvmd/pull/879) [#881](https://github.com/greenbone/gvmd/pull/881) [#883](https://github.com/greenbone/gvmd/pull/883) [#887](https://github.com/greenbone/gvmd/pull/887) [#889](https://github.com/greenbone/gvmd/pull/889) [#890](https://github.com/greenbone/gvmd/pull/890) [#891](https://github.com/greenbone/gvmd/pull/891) [#901](https://github.com/greenbone/gvmd/pull/901)
 - Change rows of built-in default filters to -2 (use "Rows Per Page" setting) [#896](https://github.com/greenbone/gvmd/pull/896)
 - Force NVT update in migrate_219_to_220 [#895](https://github.com/greenbone/gvmd/pull/895)
+- Use temp tables to speed up migrate_213_to_214 [#911](https://github.com/greenbone/gvmd/pull/911)
 
 ### Fixed
 - Consider results_trash when deleting users [#799](https://github.com/greenbone/gvmd/pull/799)
@@ -40,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix creation of "Super" permissions [#892](https://github.com/greenbone/gvmd/pull/892)
 - Setup general task preferences to launch an osp openvas task. [#898](https://github.com/greenbone/gvmd/pull/898)
 - Add tags used for result NVTs to update_nvti_cache [#916](https://github.com/greenbone/gvmd/pull/916)
+- Apply usage_type of tasks in get_aggregates (9.0) [#912](https://github.com/greenbone/gvmd/pull/912)
 
 ### Removed
 - Remove 1.3.6.1.4.1.25623.1.0.90011 from Discovery config (9.0) [#847](https://github.com/greenbone/gvmd/pull/847)

diff --git a/src/manage_migrators.c b/src/manage_migrators.c
@@ -899,6 +899,67 @@ make_tls_certificate_214 (user_t owner,
   return ret;
 }
 
+/**
+ * @brief Create temporary, indexed tables for TLS certificate host details.
+ *
+ * This makes it quicker to access host details that are slow to get repeatedly
+ * for every TLS certificate, presumably because of the "LIKE" conditions
+ * for the name or value field.
+ */
+static void
+create_temp_tables_213_to_214 ()
+{
+  // "SSLInfo", selected by a fingerprint in the value
+  sql ("CREATE TEMP TABLE report_host_details_sslinfo"
+        " (report_host integer,"
+        "  value text,"
+        "  fingerprint text,"
+        "  port text);");
+  sql ("INSERT INTO report_host_details_sslinfo"
+        "               (report_host, value, fingerprint, port)"
+        " SELECT report_host,"
+        "        value,"
+        "        (regexp_matches(value, '(.*):.*:(.*)'))[2] AS fingerprint,"
+        "        (regexp_matches(value, '(.*):.*:(.*)'))[1] AS port"
+        "  FROM report_host_details WHERE name = 'SSLInfo';");
+  sql ("CREATE INDEX"
+       " ON report_host_details_sslinfo (report_host, fingerprint)");
+
+  // "TLS/...", selected by a port number in the name
+  sql ("CREATE TEMP TABLE report_host_details_tls_ports"
+       "  (report_host integer,"
+       "   port text,"
+       "   value text)");
+  sql ("INSERT INTO report_host_details_tls_ports"
+       "              (report_host, port, value)"
+       " SELECT report_host,"
+       "        substring (name, 5) AS port,"
+       "        value"
+       "  FROM report_host_details"
+       "  WHERE name LIKE 'TLS/%%' AND name != 'TLS/port';");
+  sql ("CREATE INDEX"
+        " ON report_host_details_tls_ports (report_host)");
+
+  // "SSLDetails:...", selected by a fingerprint in the name
+  sql ("CREATE TEMP TABLE report_host_details_ssldetails"
+       "  (report_host integer,"
+       "   fingerprint text,"
+       "   value text,"
+       "   start_time integer)");
+  sql ("INSERT INTO report_host_details_ssldetails"
+       "              (report_host, fingerprint, value, start_time)"
+       " SELECT report_host,"
+       "        substring (name, 12) AS fingerprint,"
+       "        rhd.value,"
+       "        report_hosts.start_time"
+       "  FROM report_host_details AS rhd"
+       "  JOIN report_hosts"
+       "    ON report_hosts.id = rhd.report_host"
+       "  WHERE name LIKE 'SSLDetails:%%';");
+  sql ("CREATE INDEX"
+        " ON report_host_details_ssldetails (report_host, fingerprint)");
+}
+
 /**
  * @brief Migrate the database from version 213 to version 214.
  *
@@ -955,6 +1016,8 @@ migrate_213_to_214 ()
    * - The report id is last so tls_certificate_sources are created in the
    *   same order as the reports.
    */
+  create_temp_tables_213_to_214 ();
+
   init_iterator (&tls_certs,
                  "SELECT rhd.value, rhd.name, reports.owner, rhd.report_host,"
                  "       report_hosts.host, reports.uuid, rhd.source_name,"
@@ -1041,12 +1104,10 @@ migrate_213_to_214 ()
           /* Also use SSLDetails in case get_certificate_info fails
            *  or to ensure consistency with the host details */
           ssldetails
-            = sql_string ("SELECT rhd.value"
-                          " FROM report_host_details AS rhd"
-                          " JOIN report_hosts"
-                          "   ON report_hosts.id = rhd.report_host"
-                          " WHERE name = 'SSLDetails:%s'"
-                          " ORDER BY report_hosts.start_time DESC"
+            = sql_string ("SELECT value"
+                          " FROM report_host_details_ssldetails"
+                          " WHERE fingerprint = '%s'"
+                          " ORDER BY start_time DESC"
                           " LIMIT 1;",
                           quoted_scanner_fpr);
 
@@ -1094,34 +1155,32 @@ migrate_213_to_214 ()
 
       /* Collect ports for each unique certificate and owner */
       init_iterator (&ports,
-                     "SELECT value FROM report_host_details"
+                     "SELECT port FROM report_host_details_sslinfo"
                      " WHERE report_host = %llu"
-                     "   AND name = 'SSLInfo'"
-                     "   AND value LIKE '%%:%%:%s'",
+                     "   AND fingerprint = '%s'",
                      report_host,
                      quoted_scanner_fpr);
 
       has_ports = FALSE;
       while (next (&ports))
         {
-          const char *value;
-          gchar *port, *quoted_port;
+          const char *port;
+          gchar *quoted_port;
           GString *versions;
           iterator_t versions_iter;
           resource_t cert_location, cert_origin;
 
-          value = iterator_string (&ports, 0);
-          port = g_strndup (value, g_strrstr (value, ":") - value - 1);
+          port = iterator_string (&ports, 0);
           quoted_port = sql_quote (port);
 
           has_ports = TRUE;
 
           /* Collect TLS versions for each port */
           versions = g_string_new ("");
           init_iterator (&versions_iter,
-                         "SELECT value FROM report_host_details"
+                         "SELECT value FROM report_host_details_tls_ports"
                          " WHERE report_host = %llu"
-                         "   AND name = 'TLS/%s'",
+                         "   AND port = '%s'",
                          report_host,
                          quoted_port);
           while (next (&versions_iter))
@@ -1153,7 +1212,6 @@ migrate_213_to_214 ()
                timestamp,
                versions->str);
 
-          g_free (port);
           g_free (quoted_port);
           g_string_free (versions, TRUE);
         }
@@ -1172,6 +1230,10 @@ migrate_213_to_214 ()
     }
   cleanup_iterator (&tls_certs);
 
+  sql ("DROP TABLE report_host_details_sslinfo;");
+  sql ("DROP TABLE report_host_details_tls_ports;");
+  sql ("DROP TABLE report_host_details_ssldetails;");
+
   /* Set the database version to 214 */
 
   set_db_version (214);

diff --git a/src/manage_sql.c b/src/manage_sql.c
@@ -64728,17 +64728,25 @@ type_extra_where (const char *type, int trash, const char *filter,
 
   if (strcasecmp (type, "CONFIG") == 0 && extra_params)
     {
-      gchar *usage_type = g_hash_table_lookup (extra_params, "usage_type");
+      gchar *usage_type;
+      if (extra_params)
+        usage_type = g_hash_table_lookup (extra_params, "usage_type");
+      else
+        usage_type = NULL;
+
       extra_where = configs_extra_where (usage_type);
       if (extra_where == NULL)
         extra_where = g_strdup ("");
     }
   else if (strcasecmp (type, "TASK") == 0)
     {
-      if (trash)
-        extra_where = g_strdup (" AND hidden = 2");
+      gchar *usage_type;
+      if (extra_params)
+        usage_type = g_hash_table_lookup (extra_params, "usage_type");
       else
-        extra_where = g_strdup (" AND hidden = 0");
+        usage_type = NULL;
+
+      extra_where = tasks_extra_where (trash, usage_type);
     }
   else if (strcasecmp (type, "TLS_CERTIFICATE") == 0)
     {