From 289b7ce762ab587df87776bb8b6be4f38529e124 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 18 May 2020 19:20:12 +0200 Subject: [PATCH 1/6] Remove no longer affected products when updating CVE --- src/manage_sql_secinfo.c | 56 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 9509f0bab..5f9bc5c93 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2635,6 +2635,19 @@ hashed_cpes_cpe_id (GHashTable *hashed_cpes, const gchar *product_tilde) return GPOINTER_TO_INT (g_hash_table_lookup (hashed_cpes, product_tilde)); } +/** + * @brief Push a generic pointer onto an array. + * + * @param[in] array Array. + * @param[in] pointer Pointer. + */ +static void +array_remove (array_t *array, gpointer pointer) +{ + if (array) + g_ptr_array_remove_fast (array, pointer); +} + /** * @brief Insert products for a CVE. * @@ -2653,6 +2666,8 @@ insert_cve_products (element_t list, resource_t cve, element_t product; int first_product, first_affected; GString *sql_cpes, *sql_affected; + array_t *initial_affected; + iterator_t rows; if (list == NULL) return; @@ -2670,6 +2685,17 @@ insert_cve_products (element_t list, resource_t cve, " (cve, cpe)" " VALUES"); + /* Record existing affected products. */ + + initial_affected = make_array (); + init_iterator (&rows, + "SELECT cpe FROM scap.affected_products" + " WHERE cve = %llu;", + cve); + while (next (&rows)) + array_add (initial_affected, GINT_TO_POINTER (iterator_int64 (&rows, 0))); + cleanup_iterator (&rows); + /* Buffer the SQL. */ first_product = first_affected = 1; @@ -2734,15 +2760,22 @@ insert_cve_products (element_t list, resource_t cve, } else { + int cpe; + /* The product is in the db. * * So we don't need to insert it. */ + cpe = hashed_cpes_cpe_id (hashed_cpes, product_tilde); + g_string_append_printf (sql_affected, "%s (%llu, %i)", first_affected ? "" : ",", cve, - hashed_cpes_cpe_id (hashed_cpes, product_tilde)); + cpe); + + /* Remove the affected product from initial_affected. */ + array_remove (initial_affected, GINT_TO_POINTER (cpe)); } first_affected = 0; @@ -2755,6 +2788,27 @@ insert_cve_products (element_t list, resource_t cve, product = element_next (product); } + /* Remove affected_products that remain in initial_affected. */ + + if (initial_affected->len) + { + guint index; + GString *ids; + + ids = g_string_new (""); + for (index = 0; index < initial_affected->len; index++) + g_string_append_printf (ids, + "%s%i", + index == 0 ? "" : ",", + GPOINTER_TO_INT (g_ptr_array_index + (initial_affected, index))); + sql ("DELETE from scap.affected_products" + " WHERE cve = %llu AND cpe = ANY(array[%s]);", + cve, ids->str); + g_string_free (ids, TRUE); + } + g_ptr_array_free (initial_affected, TRUE); + /* Run the SQL. */ if (first_product == 0) From a3d9b887f55f0d7ed58325512a3f2ebffb4b61be Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 26 May 2020 18:03:10 +0200 Subject: [PATCH 2/6] In SCAP sync remove CVEs that were removed from XML --- src/manage_sql_secinfo.c | 46 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 5f9bc5c93..61141e175 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3077,12 +3077,14 @@ insert_cve_from_entry (element_t entry, element_t last_modified, * @param[in] last_scap_update Time of last SCAP update. * @param[in] last_cve_update Time of last update to a DFN. * @param[in] hashed_cpes Hashed CPEs. + * @param[in] all_xml_cves String to add all CVEs to. * * @return 0 nothing to do, 1 updated, -1 error. */ static int update_cve_xml (const gchar *xml_path, int last_scap_update, - int last_cve_update, GHashTable *hashed_cpes) + int last_cve_update, GHashTable *hashed_cpes, + GString *all_xml_cves) { GError *error; element_t element, entry; @@ -3141,6 +3143,7 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, { if (strcmp (element_name (entry), "entry") == 0) { + gchar *id; element_t last_modified; last_modified = element_child (entry, "vuln:last-modified-datetime"); @@ -3150,6 +3153,20 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, __FUNCTION__); goto fail; } + + id = element_attribute (entry, "id"); + if (id == NULL) + { + g_warning ("%s: id missing", + __func__); + goto fail; + } + + if (all_xml_cves->str[0] == '\0') + g_string_append_printf (all_xml_cves, "'%s'", id); + else + g_string_append_printf (all_xml_cves, ",'%s'", id); + if (parse_iso_time_element_text (last_modified) > last_cve_update) { if (insert_cve_from_entry (entry, last_modified, hashed_cpes, @@ -3193,6 +3210,7 @@ update_scap_cves (int last_scap_update) const gchar *xml_path; GHashTable *hashed_cpes; iterator_t cpes; + GString *all_xml_cves; error = NULL; dir = g_dir_open (GVM_SCAP_DATA_DIR, 0, &error); @@ -3214,13 +3232,15 @@ update_scap_cves (int last_scap_update) (gpointer*) iterator_string (&cpes, 0), GINT_TO_POINTER (iterator_int (&cpes, 1))); + all_xml_cves = g_string_new (""); + count = 0; updated_scap_cves = 0; while ((xml_path = g_dir_read_name (dir))) if (fnmatch ("nvdcve-2.0-*.xml", xml_path, 0) == 0) { switch (update_cve_xml (xml_path, last_scap_update, last_cve_update, - hashed_cpes)) + hashed_cpes, all_xml_cves)) { case 0: break; @@ -3230,12 +3250,34 @@ update_scap_cves (int last_scap_update) default: g_dir_close (dir); g_hash_table_destroy (hashed_cpes); + g_string_free (all_xml_cves, TRUE); cleanup_iterator (&cpes); return -1; } count++; } + if (strlen (all_xml_cves->str) > 0) + { + /* Remove CVES from the db that are not in all_xml_cves. */ + + g_string_prepend + (all_xml_cves, "WITH" + " removed AS (SELECT id FROM scap.cves" + " WHERE NOT uuid = ANY(array["); + + g_string_append + (all_xml_cves, "]))," + " dummy AS (DELETE FROM scap.affected_products" + " WHERE cve IN (SELECT id FROM removed)" + " RETURNING *)" + " DELETE FROM scap.cves" + " WHERE id IN (SELECT id FROM removed);"); + + sql (all_xml_cves->str); + } + g_string_free (all_xml_cves, TRUE); + if (count == 0) g_warning ("No CVEs found in %s", GVM_SCAP_DATA_DIR); From f118c477948a09cc3184273558153463677bbd22 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 19 May 2020 13:19:58 +0200 Subject: [PATCH 3/6] In SCAP sync remove CPEs that were removed from XML --- src/manage_sql_secinfo.c | 51 +++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 61141e175..d90ad5a98 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2384,11 +2384,13 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata * * @param[in] path Path to file. * @param[in] last_cve_update Time of last CVE update. + * @param[in] all_xml_cpes String to add all CPEs to. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_cpes_from_file (const gchar *path, int last_cve_update) +update_scap_cpes_from_file (const gchar *path, int last_cve_update, + GString *all_xml_cpes) { GError *error; element_t element, cpe_list, cpe_item; @@ -2448,7 +2450,7 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update) cpe_item = element_first_child (cpe_list); while (cpe_item) { - gchar *modification_date; + gchar *modification_date, *name; int modification_time; element_t item_metadata; @@ -2473,6 +2475,19 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update) goto fail; } + name = element_attribute (cpe_item, "name"); + if (name == NULL) + { + g_warning ("%s: name missing", __func__); + goto fail; + } + + if (all_xml_cpes->str[0] == '\0') + g_string_append_printf (all_xml_cpes, "'%s'", name); + else + g_string_append_printf (all_xml_cpes, ",'%s'", name); + g_free (name); + modification_time = parse_iso_time (modification_date); g_free (modification_date); @@ -2506,11 +2521,12 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update) * @brief Update SCAP CPEs. * * @param[in] last_scap_update Time of last SCAP update. + * @param[in] all_xml_cpes String to add all CPEs to. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_cpes (int last_scap_update) +update_scap_cpes (int last_scap_update, GString *all_xml_cpes) { gchar *full_path; const gchar *split_dir; @@ -2550,7 +2566,8 @@ update_scap_cpes (int last_scap_update) g_warning ("%s: Failed to split CPEs, attempting with full file", __func__); updated_scap_cpes = update_scap_cpes_from_file (full_path, - last_cve_update); + last_cve_update, + all_xml_cpes); g_free (full_path); return updated_scap_cpes; } @@ -2571,7 +2588,7 @@ update_scap_cpes (int last_scap_update) break; } - ret = update_scap_cpes_from_file (path, last_cve_update); + ret = update_scap_cpes_from_file (path, last_cve_update, all_xml_cpes); g_free (path); if (ret < 0) { @@ -5082,6 +5099,7 @@ update_scap (int lockfile, { int last_feed_update, last_scap_update; int updated_scap_ovaldefs, updated_scap_cpes, updated_scap_cves; + GString *all_xml_cpes; updated_scap_ovaldefs = 0; updated_scap_cpes = 0; @@ -5154,14 +5172,17 @@ update_scap (int lockfile, g_info ("%s: Updating data from feed", __FUNCTION__); + all_xml_cpes = g_string_new (""); + if (update_cpes) { g_debug ("%s: update cpes", __FUNCTION__); proctitle_set ("gvmd: Syncing SCAP: Updating CPEs"); - updated_scap_cpes = update_scap_cpes (last_scap_update); + updated_scap_cpes = update_scap_cpes (last_scap_update, all_xml_cpes); if (updated_scap_cpes == -1) { + g_string_free (all_xml_cpes, TRUE); manage_update_scap_db_cleanup (); goto fail; } @@ -5175,11 +5196,29 @@ update_scap (int lockfile, updated_scap_cves = update_scap_cves (last_scap_update); if (updated_scap_cves == -1) { + g_string_free (all_xml_cpes, TRUE); manage_update_scap_db_cleanup (); goto fail; } } + if (strlen (all_xml_cpes->str) > 0) + { + /* Remove CPES not in all_xml_cpes, except those in affected_products. */ + + g_string_prepend + (all_xml_cpes, "DELETE FROM scap.cpes" + " WHERE NOT EXISTS (SELECT * FROM scap.affected_products" + " WHERE cpe = scap.cpes.id)" + " AND NOT uuid = ANY(array["); + + g_string_append + (all_xml_cpes, "]);"); + + sql (all_xml_cpes->str); + } + g_string_free (all_xml_cpes, TRUE); + if (update_ovaldefs) { g_debug ("%s: update ovaldefs", __FUNCTION__); From 06e0133b0b603c0ef8b8e41e5ec1874e38ef9f59 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 19 May 2020 13:20:48 +0200 Subject: [PATCH 4/6] Free id --- src/manage_sql_secinfo.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d90ad5a98..8fe41452e 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3183,6 +3183,7 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, g_string_append_printf (all_xml_cves, "'%s'", id); else g_string_append_printf (all_xml_cves, ",'%s'", id); + g_free (id); if (parse_iso_time_element_text (last_modified) > last_cve_update) { From 9226d9a967856d7eee9a370f1245010ecece9d1e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 19 May 2020 14:44:47 +0200 Subject: [PATCH 5/6] Mark SCAP updated flags when deleting --- src/manage_sql_secinfo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 8fe41452e..c33e7b603 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3293,6 +3293,7 @@ update_scap_cves (int last_scap_update) " WHERE id IN (SELECT id FROM removed);"); sql (all_xml_cves->str); + updated_scap_cves = 1; } g_string_free (all_xml_cves, TRUE); @@ -5217,6 +5218,7 @@ update_scap (int lockfile, (all_xml_cpes, "]);"); sql (all_xml_cpes->str); + updated_scap_cpes = 1; } g_string_free (all_xml_cpes, TRUE); From 9ce12d51ddf594e112fbe32512f5e8ba1977ca9d Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 26 May 2020 18:22:22 +0200 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a754cb672..86bcfcba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Fix NVTs list in CVE details [#1098](https://github.com/greenbone/gvmd/pull/1098) +- Handle removed CPEs and CVEs in SCAP sync [#1107](https://github.com/greenbone/gvmd/pull/1107) [9.0.2]: https://github.com/greenbone/gvmd/compare/v9.0.1...gvmd-9.0