Accelerate VT feed update

CHANGELOG.md

@@ -26,7 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Request nvti_cache update only at very end of NVT update. [#426](https://github.com/greenbone/gvmd/pull/426)
 - Consolidate NVT references into unified "refs" element. [#427](https://github.com/greenbone/gvmd/pull/427)
 - Update gvm-libs version requirements to v11.0. [#480](https://github.com/greenbone/gvmd/pull/480)
--Adjust to use new API for vt references. [#526](https://github.com/greenbone/gvmd/pull/526)
+- Adjust to use new API for vt references. [#526](https://github.com/greenbone/gvmd/pull/526)
 - Expect NVT sync script in bin directory. [#546](https://github.com/greenbone/gvmd/pull/546)
 - Change internal handling of NVT XML to use nvti_t. [#562](https://github.com/greenbone/gvmd/pull/562)
 - Change NVT references like CVEs and BID to general vt_refs. [#570](https://github.com/greenbone/gvmd/pull/570) [#574](https://github.com/greenbone/gvmd/pull/574) [#582](https://github.com/greenbone/gvmd/pull/582)
@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - New columns Ports, Apps, Distance, and Auth in the CSV Hosts report format [#733](https://github.com/greenbone/gvmd/pull/733)
 - The details attribute of GET_REPORTS now defaults to 0 [#747](https://github.com/greenbone/gvmd/pull/747)
 - Incoming VT timestamps via OSP are now assumed to be seconds since epoch [#754](https://github.com/greenbone/gvmd/pull/754)
+- Accelerate NVT feed update [#757](https://github.com/greenbone/gvmd/pull/757)
 
 ### Fixed
 - A PostgreSQL statement order issue [#611](https://github.com/greenbone/gvmd/issues/611) has been addressed [#642](https://github.com/greenbone/gvmd/pull/642)

src/manage.h

@@ -1950,6 +1950,9 @@ nvt_name (const char *);
 char*
 nvts_feed_version ();
 
+time_t
+nvts_feed_version_epoch ();
+
 void
 set_nvts_feed_version (const char*);
 

src/manage_sql.c

@@ -35377,6 +35377,9 @@ new_nvts_list (event_t event, const void* event_data, alert_t alert,
   int count;
   char *details_url;
   const gchar *type;
+  time_t feed_version_epoch;
+
+  feed_version_epoch = nvts_feed_version_epoch();
 
   details_url = alert_data (alert, "method", "details_url");
   type = (gchar*) event_data;
@@ -35395,15 +35398,13 @@ new_nvts_list (event_t event, const void* event_data, alert_t alert,
   else if (event == EVENT_NEW_SECINFO)
     init_iterator (&rows,
                    "SELECT oid, name, solution_type, cvss_base, qod FROM nvts"
-                   " WHERE oid NOT IN (SELECT oid FROM old_nvts)"
-                   " ORDER BY creation_time DESC;");
+                   " WHERE creation_time > %d"
+                   " ORDER BY creation_time DESC;", (int)feed_version_epoch);
   else
     init_iterator (&rows,
                    "SELECT oid, name, solution_type, cvss_base, qod FROM nvts"
-                   " WHERE modification_time > (SELECT modification_time"
-                   "                            FROM old_nvts"
-                   "                            WHERE old_nvts.oid = nvts.oid)"
-                   " ORDER BY modification_time DESC;");
+                   " WHERE modification_time > %d"
+                   " ORDER BY modification_time DESC;", (int)feed_version_epoch);
 
   while (next (&rows))
     {

src/manage_sql_nvts.c

@@ -161,6 +161,21 @@ nvts_feed_version ()
                      sql_schema ());
 }
 
+/**
+ * @brief Return feed version of the plugins as seconds since epoch.
+ *
+ * @return Feed version in seconds since epoch of plugins.
+ */
+time_t
+nvts_feed_version_epoch ()
+{
+  struct tm tm;
+
+  memset (&tm, 0, sizeof (struct tm));
+  strptime (nvts_feed_version (), "%Y%m%d%H%M%S", &tm);
+  return mktime (&tm);
+}
+
 /**
  * @brief Set the feed version of the plugins in the plugin cache.
  *
@@ -970,37 +985,14 @@ insert_nvt_preferences_list (GList *nvt_preferences_list)
   g_list_foreach (nvt_preferences_list, insert_nvt_preference, NULL);
 }
 
-/**
- * @brief Check for new NVTs after an update.
- */
-static void
-check_for_new_nvts ()
-{
-  if (sql_int ("SELECT EXISTS"
-               " (SELECT * FROM nvts"
-               "  WHERE oid NOT IN (SELECT oid FROM old_nvts));"))
-    event (EVENT_NEW_SECINFO, "nvt", 0, 0);
-}
-
-/**
- * @brief Check for updated NVTS after an update.
- */
-static void
-check_for_updated_nvts ()
-{
-  if (sql_int ("SELECT EXISTS"
-               " (SELECT * FROM nvts"
-               "  WHERE modification_time > (SELECT modification_time"
-               "                             FROM old_nvts"
-               "                             WHERE old_nvts.oid = nvts.oid));"))
-    event (EVENT_UPDATED_SECINFO, "nvt", 0, 0);
-}
-
 /**
  * @brief Set the NVT update check time in the meta table.
+ *
+ * @param[in]  count_new       Number of new VTs with current update.
+ * @param[in]  count_modified  Number of modified VTs with current update.
  */
 static void
-set_nvts_check_time ()
+set_nvts_check_time (int count_new, int count_modified)
 {
   if (sql_int ("SELECT NOT EXISTS (SELECT * FROM meta"
                "                   WHERE name = 'nvts_check_time')"))
@@ -1012,8 +1004,12 @@ set_nvts_check_time ()
          " WHERE name = 'nvts_check_time';");
   else
     {
-      check_for_new_nvts ();
-      check_for_updated_nvts ();
+      if (count_new > 0)
+        event (EVENT_NEW_SECINFO, "nvt", 0, 0);
+
+      if (count_modified > 0)
+        event (EVENT_UPDATED_SECINFO, "nvt", 0, 0);
+
       sql ("UPDATE meta SET value = m_now ()"
            " WHERE name = 'nvts_check_time';");
     }
@@ -1302,6 +1298,13 @@ update_nvts_from_vts (entity_t *get_vts_response,
   entity_t vts, vt;
   entities_t children;
   GList *preferences;
+  int count_modified_vts, count_new_vts;
+  time_t feed_version_epoch;
+
+  count_modified_vts = 0;
+  count_new_vts = 0;
+
+  feed_version_epoch = nvts_feed_version_epoch();
 
   vts = entity_child (*get_vts_response, "vts");
   if (vts == NULL)
@@ -1331,17 +1334,17 @@ update_nvts_from_vts (entity_t *get_vts_response,
      * To solve both cases, we remove all nvt_preferences. */
     sql ("TRUNCATE nvt_preferences;");
 
-  sql ("CREATE TEMPORARY TABLE old_nvts"
-       " (oid TEXT, modification_time INTEGER);");
-  sql ("INSERT INTO old_nvts (oid, modification_time)"
-       " SELECT oid, modification_time FROM nvts;");
-
   preferences = NULL;
   children = vts->entities;
   while ((vt = first_entity (children)))
     {
       nvti_t *nvti = nvti_from_vt (vt);
 
+      if (nvti_creation_time (nvti) > feed_version_epoch)
+        count_new_vts += 1;
+      else
+        count_modified_vts += 1;
+
       insert_nvt (nvti);
 
       if (update_preferences_from_vt (vt, nvti_oid (nvti), &preferences))
@@ -1357,9 +1360,7 @@ update_nvts_from_vts (entity_t *get_vts_response,
   insert_nvt_preferences_list (preferences);
   g_list_free_full (preferences, g_free);
 
-  set_nvts_check_time ();
-
-  sql ("DROP TABLE old_nvts;");
+  set_nvts_check_time (count_new_vts, count_modified_vts);
 
   set_nvts_feed_version (scanner_feed_version);
 
@@ -1369,6 +1370,9 @@ update_nvts_from_vts (entity_t *get_vts_response,
                __FUNCTION__);
   update_all_config_caches ();
 
+  g_info ("Updating VTs in database ... %i new VTs, %i changed VTs",
+          count_new_vts, count_modified_vts);
+
   sql_commit ();
 }