From 88eb5ded48dab8cdb030e7a399226b85010674f1 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 14:09:55 +0200 Subject: [PATCH 01/25] Remove dummy functions left over from SQLite3 times --- src/manage_pg.c | 24 ------------------------ src/manage_sql.h | 4 ---- src/manage_sql_secinfo.c | 28 ---------------------------- 3 files changed, 56 deletions(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index 1f2d55eee..e7eed1353 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3305,30 +3305,6 @@ manage_db_init (const gchar *name) return 0; } -/** - * @brief Dummy function. - * - * @param[in] name Dummy arg. - */ -void -manage_db_check_mode (const gchar *name) -{ - return; -} - -/** - * @brief Dummy function. - * - * @param[in] name Dummy arg. - * - * @return 0. - */ -int -manage_db_check (const gchar *name) -{ - return 0; -} - /** * @brief Check whether CERT is available. * diff --git a/src/manage_sql.h b/src/manage_sql.h index cbc06bd88..dfa7b7b33 100644 --- a/src/manage_sql.h +++ b/src/manage_sql.h @@ -409,10 +409,6 @@ int manage_cert_db_exists (); int manage_scap_db_exists (); -void manage_db_check_mode (const gchar *); - -int manage_db_check (const gchar *); - int count (const char *, const get_data_t *, column_t *, column_t *, const char **, int, const char *, const char *, int); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 9293bc8e1..af392082c 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4693,20 +4693,6 @@ sync_cert () g_debug ("%s: sync", __func__); - manage_db_check_mode ("cert"); - - if (manage_db_check ("cert")) - { - g_warning ("%s: Database broken, resetting CERT database", - __func__); - if (manage_db_reinit ("cert")) - { - g_warning ("%s: could not reinitialize CERT database", - __func__); - goto fail; - } - } - g_info ("%s: Updating data from feed", __func__); g_debug ("%s: update dfn", __func__); @@ -5023,20 +5009,6 @@ update_scap (gboolean ignore_last_scap_update, g_debug ("%s: sync", __func__); - manage_db_check_mode ("scap"); - - if (manage_db_check ("scap")) - { - g_warning ("%s: Database broken, resetting SCAP database", - __func__); - if (manage_db_reinit ("scap")) - { - g_warning ("%s: could not reinitialize SCAP database", - __func__); - goto fail; - } - } - g_info ("%s: Updating data from feed", __func__); if (reset_required) From ec4198a18e1a4c4fc5f2b5cbeea800ceb6d4e5e6 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 17:15:06 +0200 Subject: [PATCH 02/25] In SCAP sync always update everything --- src/manage_sql_secinfo.c | 484 ++++++++------------------------------- 1 file changed, 97 insertions(+), 387 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index af392082c..68aef2766 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2328,14 +2328,11 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata * @brief Update SCAP CPEs from a file. * * @param[in] path Path to file. - * @param[in] last_cve_update Time of last CVE update. - * @param[in] all_xml_cpes String to add all CPEs to, or NULL. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_cpes_from_file (const gchar *path, int last_cve_update, - GString *all_xml_cpes) +update_scap_cpes_from_file (const gchar *path) { GError *error; element_t element, cpe_list, cpe_item; @@ -2395,7 +2392,7 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update, cpe_item = element_first_child (cpe_list); while (cpe_item) { - gchar *modification_date, *name; + gchar *modification_date; int modification_time; element_t item_metadata; @@ -2423,27 +2420,11 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update, modification_time = parse_iso_time (modification_date); g_free (modification_date); - name = element_attribute (cpe_item, "name"); - if (name == NULL) - { - g_warning ("%s: name missing", __func__); - goto fail; - } - - if (all_xml_cpes && (all_xml_cpes->str[0] == '\0')) - g_string_append_printf (all_xml_cpes, "'%s'", name); - else if (all_xml_cpes) - g_string_append_printf (all_xml_cpes, ",'%s'", name); - g_free (name); - - if (modification_time > last_cve_update) - { - if (insert_scap_cpe (&inserts, cpe_item, item_metadata, - modification_time)) - goto fail; - updated_scap_cpes = 1; - } - + if (insert_scap_cpe (&inserts, cpe_item, item_metadata, + modification_time)) + goto fail; + // FIX always true + updated_scap_cpes = 1; cpe_item = element_next (cpe_item); } @@ -2465,18 +2446,15 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update, /** * @brief Update SCAP CPEs. * - * @param[in] last_scap_update Time of last SCAP update. - * @param[in] all_xml_cpes String to add all CPEs to, or NULL. - * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_cpes (int last_scap_update, GString *all_xml_cpes) +update_scap_cpes () { gchar *full_path; const gchar *split_dir; GStatBuf state; - int updated_scap_cpes, last_cve_update, index; + int updated_scap_cpes, index; updated_scap_cpes = 0; full_path = g_build_filename (GVM_SCAP_DATA_DIR, @@ -2491,28 +2469,14 @@ update_scap_cpes (int last_scap_update, GString *all_xml_cpes) return -1; } - if ((state.st_mtime - (state.st_mtime % 60)) <= last_scap_update) - { - g_info ("Skipping CPEs, file is older than last revision" - " (this is not an error)"); - g_free (full_path); - return 0; - } - g_info ("Updating CPEs"); - /* This will be zero for an empty db, so everything will be added. */ - last_cve_update = sql_int ("SELECT max (modification_time)" - " FROM scap.cves;"); - split_dir = split_xml_file (full_path, "40Mb", ""); if (split_dir == NULL) { g_warning ("%s: Failed to split CPEs, attempting with full file", __func__); - updated_scap_cpes = update_scap_cpes_from_file (full_path, - last_cve_update, - all_xml_cpes); + updated_scap_cpes = update_scap_cpes_from_file (full_path); g_free (full_path); return updated_scap_cpes; } @@ -2533,7 +2497,7 @@ update_scap_cpes (int last_scap_update, GString *all_xml_cpes) break; } - ret = update_scap_cpes_from_file (path, last_cve_update, all_xml_cpes); + ret = update_scap_cpes_from_file (path); g_free (path); if (ret < 0) { @@ -2597,19 +2561,6 @@ hashed_cpes_cpe_id (GHashTable *hashed_cpes, const gchar *product_tilde) return GPOINTER_TO_INT (g_hash_table_lookup (hashed_cpes, product_tilde)); } -/** - * @brief Push a generic pointer onto an array. - * - * @param[in] array Array. - * @param[in] pointer Pointer. - */ -static void -array_remove (array_t *array, gpointer pointer) -{ - if (array) - g_ptr_array_remove_fast (array, pointer); -} - /** * @brief Insert products for a CVE. * @@ -2619,18 +2570,15 @@ array_remove (array_t *array, gpointer pointer) * @param[in] time_modified Time modified. * @param[in] hashed_cpes Hashed CPEs. * @param[in] transaction_size Statement counter for batching. - * @param[in] fresh Whether the db was reset. */ static void insert_cve_products (element_t list, resource_t cve, int time_modified, int time_published, - GHashTable *hashed_cpes, int *transaction_size, int fresh) + GHashTable *hashed_cpes, int *transaction_size) { element_t product; int first_product, first_affected; GString *sql_cpes, *sql_affected; - array_t *initial_affected; - iterator_t rows; if (list == NULL) return; @@ -2648,23 +2596,6 @@ insert_cve_products (element_t list, resource_t cve, " (cve, cpe)" " VALUES"); - /* Record existing affected products. */ - - if (fresh) - initial_affected = NULL; - else - { - initial_affected = make_array (); - init_iterator (&rows, - "SELECT cpe FROM scap.affected_products" - " WHERE cve = %llu;", - cve); - while (next (&rows)) - array_add (initial_affected, - GINT_TO_POINTER (iterator_int64 (&rows, 0))); - cleanup_iterator (&rows); - } - /* Buffer the SQL. */ first_product = first_affected = 1; @@ -2742,10 +2673,6 @@ insert_cve_products (element_t list, resource_t cve, "%s (%llu, %i)", first_affected ? "" : ",", cve, cpe); - - if (initial_affected) - /* Remove the affected product from initial_affected. */ - array_remove (initial_affected, GINT_TO_POINTER (cpe)); } first_affected = 0; @@ -2758,28 +2685,6 @@ insert_cve_products (element_t list, resource_t cve, product = element_next (product); } - /* Remove affected_products that remain in initial_affected. */ - - if (initial_affected && initial_affected->len) - { - guint index; - GString *ids; - - ids = g_string_new (""); - for (index = 0; index < initial_affected->len; index++) - g_string_append_printf (ids, - "%s%i", - index == 0 ? "" : ",", - GPOINTER_TO_INT (g_ptr_array_index - (initial_affected, index))); - sql ("DELETE from scap.affected_products" - " WHERE cve = %llu AND cpe = ANY(array[%s]);", - cve, ids->str); - g_string_free (ids, TRUE); - } - if (initial_affected) - g_ptr_array_free (initial_affected, TRUE); - /* Run the SQL. */ if (first_product == 0) @@ -2812,14 +2717,12 @@ insert_cve_products (element_t list, resource_t cve, * @param[in] last_modified XML last_modified element. * @param[in] transaction_size Statement counter for batching. * @param[in] hashed_cpes Hashed CPEs. - * @param[in] fresh Whether the db was reset. * * @return 0 success, -1 error. */ static int insert_cve_from_entry (element_t entry, element_t last_modified, - GHashTable *hashed_cpes, int *transaction_size, - int fresh) + GHashTable *hashed_cpes, int *transaction_size) { element_t published, summary, cvss, score, base_metrics; element_t access_vector, access_complexity, authentication; @@ -3037,7 +2940,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, g_free (score_text); insert_cve_products (list, cve, time_published, time_modified, - hashed_cpes, transaction_size, fresh); + hashed_cpes, transaction_size); g_free (quoted_id); return 0; @@ -3047,18 +2950,12 @@ insert_cve_from_entry (element_t entry, element_t last_modified, * @brief Update CVE info from a single XML feed file. * * @param[in] xml_path XML path. - * @param[in] last_scap_update Time of last SCAP update. - * @param[in] last_cve_update Time of last update to a DFN. * @param[in] hashed_cpes Hashed CPEs. - * @param[in] all_xml_cves String to add all CVEs to. - * @param[in] fresh Whether the db was reset. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_cve_xml (const gchar *xml_path, int last_scap_update, - int last_cve_update, GHashTable *hashed_cpes, - GString *all_xml_cves, int fresh) +update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) { GError *error; element_t element, entry; @@ -3079,15 +2976,6 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, return -1; } - if ((state.st_mtime - (state.st_mtime % 60)) <= last_scap_update) - { - g_info ("Skipping %s, file is older than last revision" - " (this is not an error)", - full_path); - g_free (full_path); - return 0; - } - g_info ("Updating %s", full_path); error = NULL; @@ -3117,7 +3005,6 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, { if (strcmp (element_name (entry), "entry") == 0) { - gchar *id; element_t last_modified; last_modified = element_child (entry, "vuln:last-modified-datetime"); @@ -3128,27 +3015,11 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, goto fail; } - id = element_attribute (entry, "id"); - if (id == NULL) - { - g_warning ("%s: id missing", - __func__); - goto fail; - } - - if (all_xml_cves && (all_xml_cves->str[0] == '\0')) - g_string_append_printf (all_xml_cves, "'%s'", id); - else if (all_xml_cves) - g_string_append_printf (all_xml_cves, ",'%s'", id); - g_free (id); - - if (parse_iso_time_element_text (last_modified) > last_cve_update) - { - if (insert_cve_from_entry (entry, last_modified, hashed_cpes, - &transaction_size, fresh)) - goto fail; - updated_scap_bund = 1; - } + if (insert_cve_from_entry (entry, last_modified, hashed_cpes, + &transaction_size)) + goto fail; + // FIX always true + updated_scap_bund = 1; } entry = element_next (entry); } @@ -3172,21 +3043,17 @@ update_cve_xml (const gchar *xml_path, int last_scap_update, * * Assume that the databases are attached. * - * @param[in] last_scap_update Time of last SCAP update from meta. - * @param[in] fresh Whether the db was reset. - * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_cves (int last_scap_update, int fresh) +update_scap_cves () { GError *error; - int count, last_cve_update, updated_scap_cves; + int count, updated_scap_cves; GDir *dir; const gchar *xml_path; GHashTable *hashed_cpes; iterator_t cpes; - GString *all_xml_cves; error = NULL; dir = g_dir_open (GVM_SCAP_DATA_DIR, 0, &error); @@ -3198,9 +3065,6 @@ update_scap_cves (int last_scap_update, int fresh) return -1; } - last_cve_update = sql_int ("SELECT max (modification_time)" - " FROM scap.cves;"); - hashed_cpes = g_hash_table_new (g_str_hash, g_str_equal); init_iterator (&cpes, "SELECT uuid, id FROM scap.cpes;"); while (next (&cpes)) @@ -3208,18 +3072,12 @@ update_scap_cves (int last_scap_update, int fresh) (gpointer*) iterator_string (&cpes, 0), GINT_TO_POINTER (iterator_int (&cpes, 1))); - if (fresh) - all_xml_cves = NULL; - else - all_xml_cves = g_string_new (""); - count = 0; updated_scap_cves = 0; while ((xml_path = g_dir_read_name (dir))) if (fnmatch ("nvdcve-2.0-*.xml", xml_path, 0) == 0) { - switch (update_cve_xml (xml_path, last_scap_update, last_cve_update, - hashed_cpes, all_xml_cves, fresh)) + switch (update_cve_xml (xml_path, hashed_cpes)) { case 0: break; @@ -3229,37 +3087,12 @@ update_scap_cves (int last_scap_update, int fresh) default: g_dir_close (dir); g_hash_table_destroy (hashed_cpes); - if (all_xml_cves) - g_string_free (all_xml_cves, TRUE); cleanup_iterator (&cpes); return -1; } count++; } - if (all_xml_cves && (strlen (all_xml_cves->str) > 0)) - { - /* Remove CVES from the db that are not in all_xml_cves. */ - - g_string_prepend - (all_xml_cves, "WITH" - " removed AS (SELECT id FROM scap.cves" - " WHERE NOT uuid = ANY(array["); - - g_string_append - (all_xml_cves, "]))," - " dummy AS (DELETE FROM scap.affected_products" - " WHERE cve IN (SELECT id FROM removed)" - " RETURNING *)" - " DELETE FROM scap.cves" - " WHERE id IN (SELECT id FROM removed);"); - - sql ("%s", all_xml_cves->str); - updated_scap_cves = 1; - } - if (all_xml_cves) - g_string_free (all_xml_cves, TRUE); - if (count == 0) g_warning ("No CVEs found in %s", GVM_SCAP_DATA_DIR); @@ -3515,23 +3348,19 @@ verify_oval_file (const gchar *full_path) * @brief Update OVALDEF info from a single XML feed file. * * @param[in] file_and_date Array containing XML path and timestamp. - * @param[in] last_scap_update Time of last SCAP update. - * @param[in] last_ovaldef_update Time of last update to an ovaldef. * @param[in] private Whether this is from the user's private dir. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_ovaldef_xml (gchar **file_and_date, int last_scap_update, - int last_ovaldef_update, int private) +update_ovaldef_xml (gchar **file_and_date, int private) { GError *error; element_t element, child; - const gchar *xml_path, *oval_timestamp; + const gchar *xml_path; gchar *xml_basename, *xml, *quoted_xml_basename; gsize xml_len; - GStatBuf state; - int last_oval_update, file_timestamp; + int file_timestamp; int transaction_size = 0; /* Setup variables. */ @@ -3541,26 +3370,6 @@ update_ovaldef_xml (gchar **file_and_date, int last_scap_update, g_debug ("%s: xml_path: %s", __func__, xml_path); - /* The timestamp from the OVAL XML. */ - oval_timestamp = file_and_date[1]; - - if (g_stat (xml_path, &state)) - { - g_warning ("%s: Failed to stat OVAL file %s: %s", - __func__, - xml_path, - strerror (errno)); - return -1; - } - - if ((state.st_mtime - (state.st_mtime % 60)) <= last_scap_update) - { - g_info ("Skipping %s, file is older than last revision" - " (this is not an error)", - xml_path); - return 0; - } - xml_basename = strstr (xml_path, GVM_SCAP_DATA_DIR); if (xml_basename == NULL) { @@ -3573,22 +3382,6 @@ update_ovaldef_xml (gchar **file_and_date, int last_scap_update, quoted_xml_basename = sql_quote (xml_basename); - /* The last time this file was updated in the db. */ - last_oval_update = sql_int ("SELECT max(modification_time)" - " FROM scap.ovaldefs" - " WHERE xml_file = '%s';", - quoted_xml_basename); - - if (oval_timestamp - && (parse_iso_time (oval_timestamp) <= last_oval_update)) - { - g_free (quoted_xml_basename); - g_info ("Skipping %s, file has older timestamp than latest OVAL" - " definition in database (this is not an error)", - xml_path); - return 0; - } - if (private) { /* Validate OVAL file. */ @@ -3667,11 +3460,12 @@ update_ovaldef_xml (gchar **file_and_date, int last_scap_update, &definition_date_newest, &definition_date_oldest); - if (definition_date_oldest - && (definition_date_oldest <= last_oval_update)) + if (0) { gchar *id; + // FIX remove. now always full update + id = element_attribute (definition, "id"); quoted_oval_id = sql_quote (id ? id : ""); g_free (id); @@ -4098,16 +3892,15 @@ oval_files_free () * * Assume that the databases are attached. * - * @param[in] last_scap_update Time of last SCAP update from meta. * @param[in] private Whether to update private SCAP data, instead * of the feed data. * * @return 0 nothing to do, 1 updated, -1 error. */ static int -update_scap_ovaldefs (int last_scap_update, int private) +update_scap_ovaldefs (int private) { - int count, last_oval_update, updated_scap_ovaldefs; + int count, updated_scap_ovaldefs; gchar *oval_dir; guint index; struct stat state; @@ -4244,9 +4037,6 @@ update_scap_ovaldefs (int last_scap_update, int private) /* Process each file in the list, in the sorted order. */ - last_oval_update = sql_int ("SELECT max (modification_time)" - " FROM scap.ovaldefs;"); - count = 0; updated_scap_ovaldefs = 0; for (index = 0; index < oval_files->len; index++) @@ -4254,8 +4044,7 @@ update_scap_ovaldefs (int last_scap_update, int private) gchar **pair; pair = g_ptr_array_index (oval_files, index); - switch (update_ovaldef_xml (pair, last_scap_update, last_oval_update, - private)) + switch (update_ovaldef_xml (pair, private)) { case 0: break; @@ -4328,6 +4117,7 @@ update_scap_ovaldefs (int last_scap_update, int private) } cleanup_iterator (&files); + // FIX possibly can remove too sql ("DELETE FROM scap.ovaldefs" " WHERE (xml_file NOT LIKE 'oval/%%')" "%s;", @@ -4917,174 +4707,106 @@ update_scap_placeholders (int updated_cves) } /** - * @brief Update data in the SCAP DB. + * @brief Update all data in the SCAP DB. * - * Currently only works correctly with all data or OVAL definitions. - * - * @param[in] ignore_last_scap_update Whether to ignore the last update time. - * @param[in] reset_scap_db Whether to remove the DB entirely first. - * DB will also be reset if last_scap_update - * is negative. - * @param[in] update_cpes Whether to update CPEs. - * @param[in] update_cves Whether to update CVEs. - * @param[in] update_ovaldefs Whether to update OVAL definitions. + * @param[in] reset_scap_db Whether to rebuild regardless of last_scap_update. * * @return 0 success, -1 error. */ static int -update_scap (gboolean ignore_last_scap_update, - gboolean reset_scap_db, - gboolean update_cpes, - gboolean update_cves, - gboolean update_ovaldefs) +update_scap (gboolean reset_scap_db) { - int last_feed_update, last_scap_update, reset_required; int updated_scap_ovaldefs, updated_scap_cpes, updated_scap_cves; - GString *all_xml_cpes; updated_scap_ovaldefs = 0; updated_scap_cpes = 0; updated_scap_cves = 0; - if (manage_scap_db_exists ()) - { - if (check_scap_db_version ()) - return -1; - } + if (reset_scap_db) + g_warning ("%s: Full rebuild requested, resetting SCAP db", + __func__); + else if (manage_scap_loaded () == 0) + g_warning ("%s: No SCAP db present, rebuilding SCAP db from scratch", + __func__); else { - g_info ("%s: Initializing SCAP database", __func__); + int last_scap_update; - if (manage_db_init ("scap")) - { - g_warning ("%s: Could not initialize SCAP database", __func__); - return -1; - } - } - - reset_required = reset_scap_db ? -1 : 0; - - if (manage_scap_loaded () == 0) - reset_required = -2; - else if (ignore_last_scap_update) - last_scap_update = 0; - else - { last_scap_update = sql_int ("SELECT coalesce ((SELECT value FROM scap.meta" " WHERE name = 'last_update')," " '-3');"); - if (last_scap_update < 0) - reset_required = last_scap_update; - } - - if (reset_required) - { - if (reset_required == -1) - g_warning ("%s: Full rebuild requested, resetting SCAP db", - __func__); - else if (reset_required == -2) - g_warning ("%s: No SCAP db present, rebuilding SCAP db from scratch", - __func__); - else if (reset_required == -3) + if (last_scap_update == -3) g_warning ("%s: SCAP db missing last_update record, resetting SCAP db", __func__); - else + else if (last_scap_update < 0) g_warning ("%s: Inconsistent data, resetting SCAP db", __func__); - - if (manage_db_reinit ("scap")) + else { - g_warning ("%s: could not reinitialize SCAP database", __func__); - return -1; - } - last_scap_update = 0; - } + int last_feed_update; - last_feed_update = manage_feed_timestamp ("scap"); - if (last_feed_update == -1) - return -1; + last_feed_update = manage_feed_timestamp ("scap"); - if (last_scap_update >= last_feed_update) - return -1; - - g_debug ("%s: sync", __func__); - - g_info ("%s: Updating data from feed", __func__); - - if (reset_required) - all_xml_cpes = NULL; - else - all_xml_cpes = g_string_new (""); + if (last_feed_update == -1) + return -1; - if (update_cpes) - { - g_debug ("%s: update cpes", __func__); - proctitle_set ("gvmd: Syncing SCAP: Updating CPEs"); + if (last_scap_update == last_feed_update) + { + proctitle_set ("gvmd: Syncing SCAP: done"); + return 0; + } - updated_scap_cpes = update_scap_cpes (last_scap_update, all_xml_cpes); - if (updated_scap_cpes == -1) - { - if (all_xml_cpes) - g_string_free (all_xml_cpes, TRUE); - goto fail; + if (last_scap_update > last_feed_update) + { + g_warning ("%s: last scap update later than last feed update", + __func__); + return -1; + } } } - if (update_cves) + if (manage_db_reinit ("scap")) { - g_debug ("%s: update cves", __func__); - proctitle_set ("gvmd: Syncing SCAP: Updating CVEs"); - - updated_scap_cves = update_scap_cves (last_scap_update, reset_required); - if (updated_scap_cves == -1) - { - if (all_xml_cpes) - g_string_free (all_xml_cpes, TRUE); - goto fail; - } + g_warning ("%s: could not reinitialize SCAP database", __func__); + return -1; } - if (all_xml_cpes && (strlen (all_xml_cpes->str) > 0)) - { - /* Remove CPES not in all_xml_cpes, except those in affected_products. */ + g_debug ("%s: sync", __func__); - g_string_prepend - (all_xml_cpes, "DELETE FROM scap.cpes" - " WHERE NOT EXISTS (SELECT * FROM scap.affected_products" - " WHERE cpe = scap.cpes.id)" - " AND NOT uuid = ANY(array["); + g_info ("%s: Updating data from feed", __func__); - g_string_append - (all_xml_cpes, "]);"); + g_debug ("%s: update cpes", __func__); + proctitle_set ("gvmd: Syncing SCAP: Updating CPEs"); - sql ("%s", all_xml_cpes->str); - updated_scap_cpes = 1; - } - if (all_xml_cpes) - g_string_free (all_xml_cpes, TRUE); + updated_scap_cpes = update_scap_cpes (); + if (updated_scap_cpes == -1) + goto fail; - if (update_ovaldefs) - { - g_debug ("%s: update ovaldefs", __func__); - proctitle_set ("gvmd: Syncing SCAP: Updating OVALdefs"); + g_debug ("%s: update cves", __func__); + proctitle_set ("gvmd: Syncing SCAP: Updating CVEs"); - updated_scap_ovaldefs = - update_scap_ovaldefs (last_scap_update, 0 /* Feed data. */); - if (updated_scap_ovaldefs == -1) - goto fail; + updated_scap_cves = update_scap_cves (); + if (updated_scap_cves == -1) + goto fail; - g_debug ("%s: updating user defined data", __func__); + g_debug ("%s: update ovaldefs", __func__); + proctitle_set ("gvmd: Syncing SCAP: Updating OVALdefs"); - switch (update_scap_ovaldefs (last_scap_update, 1 /* Private data. */)) - { - case 0: - break; - case -1: - goto fail; - default: - updated_scap_ovaldefs = 1; - break; - } + updated_scap_ovaldefs = update_scap_ovaldefs (0 /* Feed data. */); + if (updated_scap_ovaldefs == -1) + goto fail; + + g_debug ("%s: updating user defined data", __func__); + + switch (update_scap_ovaldefs (1 /* Private data. */)) + { + case 0: + break; + case -1: + goto fail; + default: + updated_scap_ovaldefs = 1; + break; } g_debug ("%s: update max cvss", __func__); @@ -5120,11 +4842,7 @@ update_scap (gboolean ignore_last_scap_update, static int sync_scap () { - return update_scap (FALSE, /* ignore_last_scap_update */ - FALSE, /* reset_scap_db */ - TRUE, /* update_cpes */ - TRUE, /* update_cves */ - TRUE /* update_ovaldefs */); + return update_scap (FALSE); } /** @@ -5161,11 +4879,7 @@ rebuild_scap (const char *type) if (strcasecmp (type, "all") == 0) { - ret = update_scap (TRUE, /* ignore_last_scap_update */ - TRUE, /* reset_scap_db */ - TRUE, /* update_cpes */ - TRUE, /* update_cves */ - TRUE /* update_ovaldefs */); + ret = update_scap (TRUE); if (ret == 1) ret = 2; } @@ -5173,15 +4887,11 @@ rebuild_scap (const char *type) || strcasecmp (type, "ovaldef") == 0) { g_debug ("%s: rebuilding ovaldefs", __func__); + sql ("DELETE FROM affected_ovaldefs"); sql ("DELETE FROM ovaldefs"); sql ("DELETE FROM ovalfiles"); - - ret = update_scap (TRUE, /* ignore_last_scap_update */ - FALSE, /* reset_scap_db */ - FALSE, /* update_cpes */ - FALSE, /* update_cves */ - TRUE /* update_ovaldefs */); + ret = update_scap (TRUE); if (ret == 1) ret = 2; } From 186e77bf81a34e36c70675811de24646429c30bf Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 20:30:59 +0200 Subject: [PATCH 03/25] Rebuild using a second SCAP schema --- src/manage_pg.c | 69 +++++++++++++++--------------- src/manage_sql_secinfo.c | 90 ++++++++++++++++++++-------------------- 2 files changed, 82 insertions(+), 77 deletions(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index e7eed1353..f709e6c12 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3115,32 +3115,33 @@ manage_db_init (const gchar *name) } else if (strcasecmp (name, "scap") == 0) { - sql ("CREATE OR REPLACE FUNCTION drop_scap () RETURNS void AS $$" + sql ("CREATE OR REPLACE FUNCTION drop_scap2 () RETURNS void AS $$" " BEGIN" " IF EXISTS (SELECT schema_name FROM information_schema.schemata" - " WHERE schema_name = 'scap')" + " WHERE schema_name = 'scap2')" " THEN" - " DROP SCHEMA IF EXISTS scap CASCADE;" + " DROP SCHEMA IF EXISTS scap2 CASCADE;" " END IF;" " END;" " $$ LANGUAGE plpgsql;"); - sql ("SELECT drop_scap ();"); - sql ("DROP FUNCTION drop_scap ();"); - sql ("CREATE SCHEMA scap;"); - sql ("SELECT set_config ('search_path'," - " current_setting ('search_path') || ',scap'," + " current_setting ('search_path') || ',scap2'," " false);"); + sql ("SELECT drop_scap2 ();"); + sql ("DROP FUNCTION drop_scap2 ();"); + + sql ("CREATE SCHEMA scap2;"); + /* Create tables and indexes. */ - sql ("CREATE TABLE scap.meta" + sql ("CREATE TABLE scap2.meta" " (id SERIAL PRIMARY KEY," " name text UNIQUE," " value text);"); - sql ("CREATE TABLE scap.cves" + sql ("CREATE TABLE scap2.cves" " (id SERIAL PRIMARY KEY," " uuid text UNIQUE," " name text," @@ -3157,15 +3158,15 @@ manage_db_init (const gchar *name) " products text," " cvss FLOAT DEFAULT 0);"); sql ("CREATE UNIQUE INDEX cve_idx" - " ON cves (name);"); + " ON scap2.cves (name);"); sql ("CREATE INDEX cves_by_creation_time_idx" - " ON cves (creation_time);"); + " ON scap2.cves (creation_time);"); sql ("CREATE INDEX cves_by_modification_time_idx" - " ON cves (modification_time);"); + " ON scap2.cves (modification_time);"); sql ("CREATE INDEX cves_by_cvss" - " ON cves (cvss);"); + " ON scap2.cves (cvss);"); - sql ("CREATE TABLE scap.cpes" + sql ("CREATE TABLE scap2.cpes" " (id SERIAL PRIMARY KEY," " uuid text UNIQUE," " name text," @@ -3179,28 +3180,28 @@ manage_db_init (const gchar *name) " cve_refs INTEGER DEFAULT 0," " nvd_id text);"); sql ("CREATE UNIQUE INDEX cpe_idx" - " ON cpes (name);"); + " ON scap2.cpes (name);"); sql ("CREATE INDEX cpes_by_creation_time_idx" - " ON cpes (creation_time);"); + " ON scap2.cpes (creation_time);"); sql ("CREATE INDEX cpes_by_modification_time_idx" - " ON cpes (modification_time);"); + " ON scap2.cpes (modification_time);"); sql ("CREATE INDEX cpes_by_cvss" - " ON cpes (max_cvss);"); + " ON scap2.cpes (max_cvss);"); sql ("CREATE INDEX cpes_by_uuid" - " ON cpes (uuid);"); + " ON scap2.cpes (uuid);"); - sql ("CREATE TABLE scap.affected_products" + sql ("CREATE TABLE scap2.affected_products" " (cve INTEGER NOT NULL," " cpe INTEGER NOT NULL," " UNIQUE (cve, cpe)," " FOREIGN KEY(cve) REFERENCES cves(id)," " FOREIGN KEY(cpe) REFERENCES cpes(id));"); sql ("CREATE INDEX afp_cpe_idx" - " ON affected_products (cpe);"); + " ON scap2.affected_products (cpe);"); sql ("CREATE INDEX afp_cve_idx" - " ON affected_products (cve);"); + " ON scap2.affected_products (cve);"); - sql ("CREATE TABLE scap.ovaldefs" + sql ("CREATE TABLE scap2.ovaldefs" " (id SERIAL PRIMARY KEY," " uuid text UNIQUE," " name text," /* OVAL identifier. */ @@ -3217,28 +3218,29 @@ manage_db_init (const gchar *name) " max_cvss FLOAT DEFAULT 0," " cve_refs INTEGER DEFAULT 0);"); sql ("CREATE INDEX ovaldefs_idx" - " ON ovaldefs (name);"); + " ON scap2.ovaldefs (name);"); sql ("CREATE INDEX ovaldefs_by_creation_time" - " ON ovaldefs (creation_time);"); + " ON scap2.ovaldefs (creation_time);"); - sql ("CREATE TABLE scap.ovalfiles" + sql ("CREATE TABLE scap2.ovalfiles" " (id SERIAL PRIMARY KEY," " xml_file TEXT UNIQUE);"); sql ("CREATE UNIQUE INDEX ovalfiles_idx" - " ON ovalfiles (xml_file);"); + " ON scap2.ovalfiles (xml_file);"); - sql ("CREATE TABLE scap.affected_ovaldefs" + sql ("CREATE TABLE scap2.affected_ovaldefs" " (cve INTEGER NOT NULL," " ovaldef INTEGER NOT NULL," " FOREIGN KEY(cve) REFERENCES cves(id)," " FOREIGN KEY(ovaldef) REFERENCES ovaldefs(id));"); sql ("CREATE INDEX aff_ovaldefs_def_idx" - " ON affected_ovaldefs (ovaldef);"); + " ON scap2.affected_ovaldefs (ovaldef);"); sql ("CREATE INDEX aff_ovaldefs_cve_idx" - " ON affected_ovaldefs (cve);"); + " ON scap2.affected_ovaldefs (cve);"); /* Create deletion triggers. */ +#if 0 sql ("CREATE OR REPLACE FUNCTION scap_delete_affected ()" " RETURNS TRIGGER AS $$" " BEGIN" @@ -3288,12 +3290,13 @@ manage_db_init (const gchar *name) sql ("CREATE TRIGGER affected_ovaldefs_delete" " AFTER DELETE ON affected_ovaldefs" " FOR EACH ROW EXECUTE PROCEDURE scap_update_oval ();"); +#endif /* Init tables. */ - sql ("INSERT INTO scap.meta (name, value)" + sql ("INSERT INTO scap2.meta (name, value)" " VALUES ('database_version', '16');"); - sql ("INSERT INTO scap.meta (name, value)" + sql ("INSERT INTO scap2.meta (name, value)" " VALUES ('last_update', '0');"); } else diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 68aef2766..e352b1f0c 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2376,7 +2376,7 @@ update_scap_cpes_from_file (const gchar *path) inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, - "INSERT INTO scap.cpes" + "INSERT INTO scap2.cpes" " (uuid, name, title, creation_time," " modification_time, status, deprecated_by_id," " nvd_id)" @@ -2588,11 +2588,11 @@ insert_cve_products (element_t list, resource_t cve, if (product == NULL) return; - sql_cpes = g_string_new ("INSERT INTO scap.cpes" + sql_cpes = g_string_new ("INSERT INTO scap2.cpes" " (uuid, name, creation_time," " modification_time)" " VALUES"); - sql_affected = g_string_new ("INSERT INTO scap.affected_products" + sql_affected = g_string_new ("INSERT INTO scap2.affected_products" " (cve, cpe)" " VALUES"); @@ -2893,7 +2893,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, time_published = parse_iso_time_element_text (published); score_text = score ? element_text (score) : g_strdup ("NULL"); cve = sql_int64_0 - ("INSERT INTO scap.cves" + ("INSERT INTO scap2.cves" " (uuid, name, creation_time, modification_time," " cvss, description, vector, complexity," " authentication, confidentiality_impact," @@ -2915,7 +2915,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, " integrity_impact = EXCLUDED.integrity_impact," " availability_impact = EXCLUDED.availability_impact," " products = EXCLUDED.products" - " RETURNING scap.cves.id;", + " RETURNING scap2.cves.id;", quoted_id, quoted_id, time_published, @@ -3066,7 +3066,7 @@ update_scap_cves () } hashed_cpes = g_hash_table_new (g_str_hash, g_str_equal); - init_iterator (&cpes, "SELECT uuid, id FROM scap.cpes;"); + init_iterator (&cpes, "SELECT uuid, id FROM scap2.cpes;"); while (next (&cpes)) g_hash_table_insert (hashed_cpes, (gpointer*) iterator_string (&cpes, 0), @@ -3584,7 +3584,7 @@ update_ovaldef_xml (gchar **file_and_date, int private) quoted_status = sql_quote (""); g_free (status_text); - sql ("INSERT INTO scap.ovaldefs" + sql ("INSERT INTO scap2.ovaldefs" " (uuid, name, comment, creation_time," " modification_time, version, deprecated, def_class," " title, description, xml_file, status," @@ -4103,7 +4103,7 @@ update_scap_ovaldefs (int private) g_string_append (oval_files_clause, "))"); init_iterator (&files, - "SELECT DISTINCT xml_file FROM scap.ovaldefs" + "SELECT DISTINCT xml_file FROM scap2.ovaldefs" " WHERE (xml_file NOT LIKE 'oval/%%')" "%s", oval_files_clause->str); @@ -4118,7 +4118,7 @@ update_scap_ovaldefs (int private) cleanup_iterator (&files); // FIX possibly can remove too - sql ("DELETE FROM scap.ovaldefs" + sql ("DELETE FROM scap2.ovaldefs" " WHERE (xml_file NOT LIKE 'oval/%%')" "%s;", oval_files_clause->str); @@ -4385,7 +4385,7 @@ update_cvss_dfn_cert (int updated_dfn_cert, int last_cert_update, sql_recursive_triggers_off (); sql ("UPDATE cert.dfn_cert_advs" " SET max_cvss = (SELECT max (cvss)" - " FROM scap.cves" + " FROM scap2.cves" " WHERE name" " IN (SELECT cve_name" " FROM cert.dfn_cert_cves" @@ -4417,7 +4417,7 @@ update_cvss_cert_bund (int updated_cert_bund, int last_cert_update, sql_recursive_triggers_off (); sql ("UPDATE cert.cert_bund_advs" " SET max_cvss = (SELECT max (cvss)" - " FROM scap.cves" + " FROM scap2.cves" " WHERE name" " IN (SELECT cve_name" " FROM cert.cert_bund_cves" @@ -4501,7 +4501,7 @@ sync_cert () last_scap_update = 0; if (manage_scap_loaded ()) - last_scap_update = sql_int ("SELECT coalesce ((SELECT value FROM scap.meta" + last_scap_update = sql_int ("SELECT coalesce ((SELECT value FROM scap2.meta" " WHERE name = 'last_update')," " '0');"); g_debug ("%s: last_scap_update: %i", __func__, last_scap_update); @@ -4546,6 +4546,7 @@ manage_sync_cert (sigset_t *sigmask_current) int check_scap_db_version () { + // FIX drop with full rebuild switch (manage_scap_db_version ()) { /* TODO The sync script had a whole lot of migrators in here. */ @@ -4568,8 +4569,8 @@ check_scap_db_version () return manage_db_reinit ("scap"); break; case 15: - sql ("ALTER TABLE scap.affected_products ADD UNIQUE (cve, cpe);"); - sql ("UPDATE scap.meta" + sql ("ALTER TABLE scap2.affected_products ADD UNIQUE (cve, cpe);"); + sql ("UPDATE scap2.meta" " SET value = '16'" " WHERE name = 'database_version';"); break; @@ -4625,7 +4626,7 @@ update_scap_timestamp () } g_debug ("%s: setting last_update: %lld", __func__, (long long) stamp); - sql ("UPDATE scap.meta SET value = '%lld' WHERE name = 'last_update';", + sql ("UPDATE scap2.meta SET value = '%lld' WHERE name = 'last_update';", (long long) stamp); return 0; @@ -4647,12 +4648,12 @@ update_scap_cvss (int updated_cves, int updated_cpes, int updated_ovaldefs) { g_info ("Updating CVSS scores and CVE counts for CPEs"); sql_recursive_triggers_off (); - sql ("UPDATE scap.cpes" + sql ("UPDATE scap2.cpes" " SET (max_cvss, cve_refs)" " = (WITH affected_cves" - " AS (SELECT cve FROM scap.affected_products" + " AS (SELECT cve FROM scap2.affected_products" " WHERE cpe=cpes.id)" - " SELECT (SELECT max (cvss) FROM scap.cves" + " SELECT (SELECT max (cvss) FROM scap2.cves" " WHERE id IN (SELECT cve FROM affected_cves))," " (SELECT count (*) FROM affected_cves));"); } @@ -4663,11 +4664,11 @@ update_scap_cvss (int updated_cves, int updated_cpes, int updated_ovaldefs) { g_info ("Updating CVSS scores for OVAL definitions"); sql_recursive_triggers_off (); - sql ("UPDATE scap.ovaldefs" + sql ("UPDATE scap2.ovaldefs" " SET max_cvss = (SELECT max (cvss)" - " FROM scap.cves" + " FROM scap2.cves" " WHERE id IN (SELECT cve" - " FROM scap.affected_ovaldefs" + " FROM scap2.affected_ovaldefs" " WHERE ovaldef=ovaldefs.id)" " AND cvss != 0.0);"); } @@ -4689,16 +4690,16 @@ update_scap_placeholders (int updated_cves) if (updated_cves) { g_info ("Updating placeholder CPEs"); - sql ("UPDATE scap.cpes" + sql ("UPDATE scap2.cpes" " SET creation_time = (SELECT min (creation_time)" - " FROM scap.cves" + " FROM scap2.cves" " WHERE id IN (SELECT cve" - " FROM scap.affected_products" + " FROM scap2.affected_products" " WHERE cpe=cpes.id))," " modification_time = (SELECT min(creation_time)" - " FROM scap.cves" + " FROM scap2.cves" " WHERE id IN (SELECT cve" - " FROM scap.affected_products" + " FROM scap2.affected_products" " WHERE cpe=cpes.id))" " WHERE cpes.title IS NULL;"); } @@ -4765,12 +4766,16 @@ update_scap (gboolean reset_scap_db) } } - if (manage_db_reinit ("scap")) + /* Create a new schema, "scap2". */ + + if (manage_db_init ("scap")) { - g_warning ("%s: could not reinitialize SCAP database", __func__); + g_warning ("%s: could not initialize SCAP database 2", __func__); return -1; } + /* Update into the new schema. */ + g_debug ("%s: sync", __func__); g_info ("%s: Updating data from feed", __func__); @@ -4825,6 +4830,19 @@ update_scap (gboolean reset_scap_db) if (update_scap_timestamp ()) goto fail; + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'name');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + g_info ("%s: Updating SCAP info succeeded", __func__); proctitle_set ("gvmd: Syncing SCAP: done"); @@ -4929,22 +4947,6 @@ manage_rebuild_scap (GSList *log_config, const gchar *database, if (ret) return -1; - if (manage_scap_db_exists ()) - { - if (check_scap_db_version ()) - goto fail; - } - else - { - g_info ("%s: Initializing SCAP database", __func__); - - if (manage_db_init ("scap")) - { - g_warning ("%s: Could not initialize SCAP database", __func__); - goto fail; - } - } - ret = rebuild_scap (type); if (ret == 1) { From 1efe48ed25547d5672a5fc3e8c30d40c483bdfbf Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 20:50:10 +0200 Subject: [PATCH 04/25] Correct schema name --- src/manage_sql_secinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index e352b1f0c..6958e295a 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4834,7 +4834,7 @@ update_scap (gboolean reset_scap_db) if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" " information_schema.schemata" - " WHERE schema_name = 'name');")) + " WHERE schema_name = 'scap');")) { sql ("ALTER SCHEMA scap RENAME TO scap3;"); sql ("ALTER SCHEMA scap2 RENAME TO scap;"); From 68b45ea8ac3b529b90588120c1b15102677ce97a Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 21:09:08 +0200 Subject: [PATCH 05/25] Make the SCAP indexes after the data is inserted --- src/manage_pg.c | 93 ++++++++++++++++++++++++++-------------- src/manage_sql_secinfo.c | 16 +++++++ 2 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index f709e6c12..2ca69959d 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3134,7 +3134,7 @@ manage_db_init (const gchar *name) sql ("CREATE SCHEMA scap2;"); - /* Create tables and indexes. */ + /* Create tables. */ sql ("CREATE TABLE scap2.meta" " (id SERIAL PRIMARY KEY," @@ -3157,14 +3157,6 @@ manage_db_init (const gchar *name) " availability_impact text," " products text," " cvss FLOAT DEFAULT 0);"); - sql ("CREATE UNIQUE INDEX cve_idx" - " ON scap2.cves (name);"); - sql ("CREATE INDEX cves_by_creation_time_idx" - " ON scap2.cves (creation_time);"); - sql ("CREATE INDEX cves_by_modification_time_idx" - " ON scap2.cves (modification_time);"); - sql ("CREATE INDEX cves_by_cvss" - " ON scap2.cves (cvss);"); sql ("CREATE TABLE scap2.cpes" " (id SERIAL PRIMARY KEY," @@ -3179,16 +3171,6 @@ manage_db_init (const gchar *name) " max_cvss FLOAT DEFAULT 0," " cve_refs INTEGER DEFAULT 0," " nvd_id text);"); - sql ("CREATE UNIQUE INDEX cpe_idx" - " ON scap2.cpes (name);"); - sql ("CREATE INDEX cpes_by_creation_time_idx" - " ON scap2.cpes (creation_time);"); - sql ("CREATE INDEX cpes_by_modification_time_idx" - " ON scap2.cpes (modification_time);"); - sql ("CREATE INDEX cpes_by_cvss" - " ON scap2.cpes (max_cvss);"); - sql ("CREATE INDEX cpes_by_uuid" - " ON scap2.cpes (uuid);"); sql ("CREATE TABLE scap2.affected_products" " (cve INTEGER NOT NULL," @@ -3196,10 +3178,6 @@ manage_db_init (const gchar *name) " UNIQUE (cve, cpe)," " FOREIGN KEY(cve) REFERENCES cves(id)," " FOREIGN KEY(cpe) REFERENCES cpes(id));"); - sql ("CREATE INDEX afp_cpe_idx" - " ON scap2.affected_products (cpe);"); - sql ("CREATE INDEX afp_cve_idx" - " ON scap2.affected_products (cve);"); sql ("CREATE TABLE scap2.ovaldefs" " (id SERIAL PRIMARY KEY," @@ -3217,26 +3195,16 @@ manage_db_init (const gchar *name) " status TEXT," " max_cvss FLOAT DEFAULT 0," " cve_refs INTEGER DEFAULT 0);"); - sql ("CREATE INDEX ovaldefs_idx" - " ON scap2.ovaldefs (name);"); - sql ("CREATE INDEX ovaldefs_by_creation_time" - " ON scap2.ovaldefs (creation_time);"); sql ("CREATE TABLE scap2.ovalfiles" " (id SERIAL PRIMARY KEY," " xml_file TEXT UNIQUE);"); - sql ("CREATE UNIQUE INDEX ovalfiles_idx" - " ON scap2.ovalfiles (xml_file);"); sql ("CREATE TABLE scap2.affected_ovaldefs" " (cve INTEGER NOT NULL," " ovaldef INTEGER NOT NULL," " FOREIGN KEY(cve) REFERENCES cves(id)," " FOREIGN KEY(ovaldef) REFERENCES ovaldefs(id));"); - sql ("CREATE INDEX aff_ovaldefs_def_idx" - " ON scap2.affected_ovaldefs (ovaldef);"); - sql ("CREATE INDEX aff_ovaldefs_cve_idx" - " ON scap2.affected_ovaldefs (cve);"); /* Create deletion triggers. */ @@ -3308,6 +3276,65 @@ manage_db_init (const gchar *name) return 0; } +/** + * @brief Init external database. + * + * @param[in] name Name. Currently only "scap". + * + * @return 0 success, -1 error. + */ +int +manage_db_init_indexes (const gchar *name) +{ + if (strcasecmp (name, "scap") == 0) + { + sql ("CREATE UNIQUE INDEX cve_idx" + " ON scap2.cves (name);"); + sql ("CREATE INDEX cves_by_creation_time_idx" + " ON scap2.cves (creation_time);"); + sql ("CREATE INDEX cves_by_modification_time_idx" + " ON scap2.cves (modification_time);"); + sql ("CREATE INDEX cves_by_cvss" + " ON scap2.cves (cvss);"); + + sql ("CREATE UNIQUE INDEX cpe_idx" + " ON scap2.cpes (name);"); + sql ("CREATE INDEX cpes_by_creation_time_idx" + " ON scap2.cpes (creation_time);"); + sql ("CREATE INDEX cpes_by_modification_time_idx" + " ON scap2.cpes (modification_time);"); + sql ("CREATE INDEX cpes_by_cvss" + " ON scap2.cpes (max_cvss);"); + sql ("CREATE INDEX cpes_by_uuid" + " ON scap2.cpes (uuid);"); + + sql ("CREATE INDEX afp_cpe_idx" + " ON scap2.affected_products (cpe);"); + sql ("CREATE INDEX afp_cve_idx" + " ON scap2.affected_products (cve);"); + + sql ("CREATE INDEX ovaldefs_idx" + " ON scap2.ovaldefs (name);"); + sql ("CREATE INDEX ovaldefs_by_creation_time" + " ON scap2.ovaldefs (creation_time);"); + + sql ("CREATE UNIQUE INDEX ovalfiles_idx" + " ON scap2.ovalfiles (xml_file);"); + + sql ("CREATE INDEX aff_ovaldefs_def_idx" + " ON scap2.affected_ovaldefs (ovaldef);"); + sql ("CREATE INDEX aff_ovaldefs_cve_idx" + " ON scap2.affected_ovaldefs (cve);"); + } + else + { + assert (0); + return -1; + } + + return 0; +} + /** * @brief Check whether CERT is available. * diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 6958e295a..fab57abed 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -77,6 +77,9 @@ manage_db_remove (const gchar *); int manage_db_init (const gchar *); +int +manage_db_init_indexes (const gchar *); + /* Helpers. */ @@ -4814,6 +4817,19 @@ update_scap (gboolean reset_scap_db) break; } + /* Add the indexes now that the data is ready. */ + + g_debug ("%s: update max cvss", __func__); + proctitle_set ("gvmd: Syncing SCAP: Updating max CVSS"); + + if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not initialize SCAP indexes", __func__); + return -1; + } + + /* Do calculations that need all data. */ + g_debug ("%s: update max cvss", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating max CVSS"); From e6d27a602c49d2693a738f52b3caf9ac153ce951 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 21:31:12 +0200 Subject: [PATCH 06/25] Remove flags, they are always updated now --- src/manage_sql_secinfo.c | 91 ++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 54 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index fab57abed..613653583 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4643,41 +4643,30 @@ update_scap_timestamp () * @param[in] updated_ovaldefs Whether OVAL defs were updated. */ static void -update_scap_cvss (int updated_cves, int updated_cpes, int updated_ovaldefs) +update_scap_cvss () { /* TODO greenbone-scapdata-sync did retries. */ - if (updated_cves || updated_cpes) - { - g_info ("Updating CVSS scores and CVE counts for CPEs"); - sql_recursive_triggers_off (); - sql ("UPDATE scap2.cpes" - " SET (max_cvss, cve_refs)" - " = (WITH affected_cves" - " AS (SELECT cve FROM scap2.affected_products" - " WHERE cpe=cpes.id)" - " SELECT (SELECT max (cvss) FROM scap2.cves" - " WHERE id IN (SELECT cve FROM affected_cves))," - " (SELECT count (*) FROM affected_cves));"); - } - else - g_info ("No CPEs or CVEs updated, skipping CVSS and CVE recount for CPEs."); - - if (updated_cves || updated_ovaldefs) - { - g_info ("Updating CVSS scores for OVAL definitions"); - sql_recursive_triggers_off (); - sql ("UPDATE scap2.ovaldefs" - " SET max_cvss = (SELECT max (cvss)" - " FROM scap2.cves" - " WHERE id IN (SELECT cve" - " FROM scap2.affected_ovaldefs" - " WHERE ovaldef=ovaldefs.id)" - " AND cvss != 0.0);"); - } - else - g_info ("No OVAL definitions or CVEs updated," - " skipping CVSS recount for OVAL definitions."); + g_info ("Updating CVSS scores and CVE counts for CPEs"); + sql_recursive_triggers_off (); + sql ("UPDATE scap2.cpes" + " SET (max_cvss, cve_refs)" + " = (WITH affected_cves" + " AS (SELECT cve FROM scap2.affected_products" + " WHERE cpe=cpes.id)" + " SELECT (SELECT max (cvss) FROM scap2.cves" + " WHERE id IN (SELECT cve FROM affected_cves))," + " (SELECT count (*) FROM affected_cves));"); + + g_info ("Updating CVSS scores for OVAL definitions"); + sql_recursive_triggers_off (); + sql ("UPDATE scap2.ovaldefs" + " SET max_cvss = (SELECT max (cvss)" + " FROM scap2.cves" + " WHERE id IN (SELECT cve" + " FROM scap2.affected_ovaldefs" + " WHERE ovaldef=ovaldefs.id)" + " AND cvss != 0.0);"); } /** @@ -4686,28 +4675,23 @@ update_scap_cvss (int updated_cves, int updated_cpes, int updated_ovaldefs) * @param[in] updated_cves Whether the CVEs were updated. */ static void -update_scap_placeholders (int updated_cves) +update_scap_placeholders () { /* TODO greenbone-scapdata-sync did retries. */ - if (updated_cves) - { - g_info ("Updating placeholder CPEs"); - sql ("UPDATE scap2.cpes" - " SET creation_time = (SELECT min (creation_time)" - " FROM scap2.cves" - " WHERE id IN (SELECT cve" - " FROM scap2.affected_products" - " WHERE cpe=cpes.id))," - " modification_time = (SELECT min(creation_time)" - " FROM scap2.cves" - " WHERE id IN (SELECT cve" - " FROM scap2.affected_products" - " WHERE cpe=cpes.id))" - " WHERE cpes.title IS NULL;"); - } - else - g_info ("No CVEs updated, skipping placeholder CPE update."); + g_info ("Updating placeholder CPEs"); + sql ("UPDATE scap2.cpes" + " SET creation_time = (SELECT min (creation_time)" + " FROM scap2.cves" + " WHERE id IN (SELECT cve" + " FROM scap2.affected_products" + " WHERE cpe=cpes.id))," + " modification_time = (SELECT min(creation_time)" + " FROM scap2.cves" + " WHERE id IN (SELECT cve" + " FROM scap2.affected_products" + " WHERE cpe=cpes.id))" + " WHERE cpes.title IS NULL;"); } /** @@ -4833,13 +4817,12 @@ update_scap (gboolean reset_scap_db) g_debug ("%s: update max cvss", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating max CVSS"); - update_scap_cvss (updated_scap_cves, updated_scap_cpes, - updated_scap_ovaldefs); + update_scap_cvss (); g_debug ("%s: update placeholders", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating placeholders"); - update_scap_placeholders (updated_scap_cves); + update_scap_placeholders (); g_debug ("%s: update timestamp", __func__); From ac0e21a94113a9f624d4a1ab13a6a683d333757e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 22:40:29 +0200 Subject: [PATCH 07/25] Remove updated returns, as everything updates now --- src/manage_sql_secinfo.c | 67 ++++++++++++---------------------------- 1 file changed, 20 insertions(+), 47 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 613653583..28a0d774b 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2332,7 +2332,7 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata * * @param[in] path Path to file. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_scap_cpes_from_file (const gchar *path) @@ -2341,13 +2341,10 @@ update_scap_cpes_from_file (const gchar *path) element_t element, cpe_list, cpe_item; gchar *xml; gsize xml_len; - int updated_scap_cpes; inserts_t inserts; g_debug ("%s: parsing %s", __func__, path); - updated_scap_cpes = 0; - error = NULL; g_file_get_contents (path, &xml, &xml_len, &error); if (error) @@ -2426,8 +2423,6 @@ update_scap_cpes_from_file (const gchar *path) if (insert_scap_cpe (&inserts, cpe_item, item_metadata, modification_time)) goto fail; - // FIX always true - updated_scap_cpes = 1; cpe_item = element_next (cpe_item); } @@ -2436,7 +2431,7 @@ update_scap_cpes_from_file (const gchar *path) inserts_run (&inserts); sql_commit (); - return updated_scap_cpes; + return 0; fail: inserts_free (&inserts); @@ -2449,7 +2444,7 @@ update_scap_cpes_from_file (const gchar *path) /** * @brief Update SCAP CPEs. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_scap_cpes () @@ -2457,9 +2452,8 @@ update_scap_cpes () gchar *full_path; const gchar *split_dir; GStatBuf state; - int updated_scap_cpes, index; + int index; - updated_scap_cpes = 0; full_path = g_build_filename (GVM_SCAP_DATA_DIR, "official-cpe-dictionary_v2.2.xml", NULL); @@ -2477,11 +2471,13 @@ update_scap_cpes () split_dir = split_xml_file (full_path, "40Mb", ""); if (split_dir == NULL) { + int ret; + g_warning ("%s: Failed to split CPEs, attempting with full file", __func__); - updated_scap_cpes = update_scap_cpes_from_file (full_path); + ret = update_scap_cpes_from_file (full_path); g_free (full_path); - return updated_scap_cpes; + return ret; } g_free (full_path); @@ -2507,13 +2503,11 @@ update_scap_cpes () gvm_file_remove_recurse (split_dir); return -1; } - if (ret) - updated_scap_cpes = 1; } gvm_file_remove_recurse (split_dir); - return updated_scap_cpes; + return 0; } @@ -3046,13 +3040,13 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) * * Assume that the databases are attached. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_scap_cves () { GError *error; - int count, updated_scap_cves; + int count; GDir *dir; const gchar *xml_path; GHashTable *hashed_cpes; @@ -3076,7 +3070,6 @@ update_scap_cves () GINT_TO_POINTER (iterator_int (&cpes, 1))); count = 0; - updated_scap_cves = 0; while ((xml_path = g_dir_read_name (dir))) if (fnmatch ("nvdcve-2.0-*.xml", xml_path, 0) == 0) { @@ -3085,7 +3078,6 @@ update_scap_cves () case 0: break; case 1: - updated_scap_cves = 1; break; default: g_dir_close (dir); @@ -3102,7 +3094,7 @@ update_scap_cves () g_dir_close (dir); g_hash_table_destroy (hashed_cpes); cleanup_iterator (&cpes); - return updated_scap_cves; + return 0; } @@ -3898,12 +3890,12 @@ oval_files_free () * @param[in] private Whether to update private SCAP data, instead * of the feed data. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_scap_ovaldefs (int private) { - int count, updated_scap_ovaldefs; + int count; gchar *oval_dir; guint index; struct stat state; @@ -4041,7 +4033,6 @@ update_scap_ovaldefs (int private) /* Process each file in the list, in the sorted order. */ count = 0; - updated_scap_ovaldefs = 0; for (index = 0; index < oval_files->len; index++) { gchar **pair; @@ -4052,7 +4043,6 @@ update_scap_ovaldefs (int private) case 0: break; case 1: - updated_scap_ovaldefs = 1; break; default: oval_files_free (); @@ -4133,7 +4123,7 @@ update_scap_ovaldefs (int private) g_free (oval_dir); oval_files_free (); - return updated_scap_ovaldefs; + return 0; } @@ -4704,12 +4694,6 @@ update_scap_placeholders () static int update_scap (gboolean reset_scap_db) { - int updated_scap_ovaldefs, updated_scap_cpes, updated_scap_cves; - - updated_scap_ovaldefs = 0; - updated_scap_cpes = 0; - updated_scap_cves = 0; - if (reset_scap_db) g_warning ("%s: Full rebuild requested, resetting SCAP db", __func__); @@ -4770,36 +4754,25 @@ update_scap (gboolean reset_scap_db) g_debug ("%s: update cpes", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating CPEs"); - updated_scap_cpes = update_scap_cpes (); - if (updated_scap_cpes == -1) + if (update_scap_cpes () == -1) goto fail; g_debug ("%s: update cves", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating CVEs"); - updated_scap_cves = update_scap_cves (); - if (updated_scap_cves == -1) + if (update_scap_cves () == -1) goto fail; g_debug ("%s: update ovaldefs", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating OVALdefs"); - updated_scap_ovaldefs = update_scap_ovaldefs (0 /* Feed data. */); - if (updated_scap_ovaldefs == -1) + if (update_scap_ovaldefs (0 /* Feed data. */) == -1) goto fail; g_debug ("%s: updating user defined data", __func__); - switch (update_scap_ovaldefs (1 /* Private data. */)) - { - case 0: - break; - case -1: - goto fail; - default: - updated_scap_ovaldefs = 1; - break; - } + if (update_scap_ovaldefs (1 /* Private data. */) == -1) + goto fail; /* Add the indexes now that the data is ready. */ From 2741bd55b5d5dc962659f614964dfaa1e7de594f Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 22:42:15 +0200 Subject: [PATCH 08/25] Merge OVAL rebuild in all case --- src/manage_sql_secinfo.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 28a0d774b..ccbf14eef 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4867,24 +4867,14 @@ rebuild_scap (const char *type) else if (ret) return -1; - if (strcasecmp (type, "all") == 0) + if (strcasecmp (type, "all") == 0 + || strcasecmp (type, "ovaldefs") == 0 + || strcasecmp (type, "ovaldef") == 0) { ret = update_scap (TRUE); if (ret == 1) ret = 2; } - else if (strcasecmp (type, "ovaldefs") == 0 - || strcasecmp (type, "ovaldef") == 0) - { - g_debug ("%s: rebuilding ovaldefs", __func__); - - sql ("DELETE FROM affected_ovaldefs"); - sql ("DELETE FROM ovaldefs"); - sql ("DELETE FROM ovalfiles"); - ret = update_scap (TRUE); - if (ret == 1) - ret = 2; - } else ret = 1; From f02af10ba1bb8a8c2b710ce86f61a191ae5fb6ea Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 22:44:25 +0200 Subject: [PATCH 09/25] Return empty trigger function --- src/manage_sql_secinfo.c | 4 ---- src/sql_pg.c | 11 ----------- 2 files changed, 15 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index ccbf14eef..56e221301 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4375,7 +4375,6 @@ update_cvss_dfn_cert (int updated_dfn_cert, int last_cert_update, if (updated_dfn_cert || (last_scap_update > last_cert_update)) { g_info ("Updating Max CVSS for DFN-CERT"); - sql_recursive_triggers_off (); sql ("UPDATE cert.dfn_cert_advs" " SET max_cvss = (SELECT max (cvss)" " FROM scap2.cves" @@ -4407,7 +4406,6 @@ update_cvss_cert_bund (int updated_cert_bund, int last_cert_update, if (updated_cert_bund || (last_scap_update > last_cert_update)) { g_info ("Updating Max CVSS for CERT-Bund"); - sql_recursive_triggers_off (); sql ("UPDATE cert.cert_bund_advs" " SET max_cvss = (SELECT max (cvss)" " FROM scap2.cves" @@ -4638,7 +4636,6 @@ update_scap_cvss () /* TODO greenbone-scapdata-sync did retries. */ g_info ("Updating CVSS scores and CVE counts for CPEs"); - sql_recursive_triggers_off (); sql ("UPDATE scap2.cpes" " SET (max_cvss, cve_refs)" " = (WITH affected_cves" @@ -4649,7 +4646,6 @@ update_scap_cvss () " (SELECT count (*) FROM affected_cves));"); g_info ("Updating CVSS scores for OVAL definitions"); - sql_recursive_triggers_off (); sql ("UPDATE scap2.ovaldefs" " SET max_cvss = (SELECT max (cvss)" " FROM scap2.cves" diff --git a/src/sql_pg.c b/src/sql_pg.c index 2483fcee2..9346bb840 100644 --- a/src/sql_pg.c +++ b/src/sql_pg.c @@ -231,17 +231,6 @@ sql_default_database () return "gvmd"; } -/** - * @brief Turn off recursive triggers. - * - * Ignored when DB is Postgres. - */ -void -sql_recursive_triggers_off () -{ - return; -} - /** * @brief Open the database. * From 05b27f1407fa4d720175308bbbb4f09831e3d42e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Thu, 28 May 2020 22:55:36 +0200 Subject: [PATCH 10/25] Move index creation back up, as the data load needs them --- src/manage_sql_secinfo.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 56e221301..472a60c44 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4741,6 +4741,17 @@ update_scap (gboolean reset_scap_db) return -1; } + /* Add the indexes, now that the data is ready. */ + + g_debug ("%s: add indexes", __func__); + proctitle_set ("gvmd: Syncing SCAP: Adding indexes"); + + if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not initialize SCAP indexes", __func__); + return -1; + } + /* Update into the new schema. */ g_debug ("%s: sync", __func__); @@ -4770,17 +4781,6 @@ update_scap (gboolean reset_scap_db) if (update_scap_ovaldefs (1 /* Private data. */) == -1) goto fail; - /* Add the indexes now that the data is ready. */ - - g_debug ("%s: update max cvss", __func__); - proctitle_set ("gvmd: Syncing SCAP: Updating max CVSS"); - - if (manage_db_init_indexes ("scap")) - { - g_warning ("%s: could not initialize SCAP indexes", __func__); - return -1; - } - /* Do calculations that need all data. */ g_debug ("%s: update max cvss", __func__); From 0f23980b7bb69f67a2fcfecd96bcbca7eacd4d74 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Fri, 29 May 2020 10:29:07 +0200 Subject: [PATCH 11/25] Add start of optional CSV loading --- src/manage_sql_secinfo.c | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 472a60c44..d37e12023 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4741,6 +4741,87 @@ update_scap (gboolean reset_scap_db) return -1; } + gchar *file_cve, *file_cpe, *file_affected_products; + gchar *file_ovaldefs, *file_ovalfiles, *file_affected_ovaldefs; + + // GRANT pg_read_server_files TO student; +/* +psql -c "COPY scap.cpes TO STDOUT WITH CSV" gvmd > table-cpes.csv +psql -c "COPY scap.cves TO STDOUT WITH CSV" gvmd > table-cves.csv +psql -c "COPY scap.affected_products TO STDOUT WITH CSV" gvmd > table-affected-products.csv +psql -c "COPY scap.ovaldefs TO STDOUT WITH CSV" gvmd > table-ovaldefs.csv +psql -c "COPY scap.ovalfiles TO STDOUT WITH CSV" gvmd > table-ovalfiles.csv +psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-ovaldefs.csv +*/ + + file_cve = g_build_filename (GVM_SCAP_DATA_DIR, "table-cves.csv", NULL); + file_cpe = g_build_filename (GVM_SCAP_DATA_DIR, "table-cpes.csv", NULL); + file_affected_products = g_build_filename (GVM_SCAP_DATA_DIR, + "table-affected.csv", + NULL); + file_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovaldefs.csv", NULL); + file_ovalfiles = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovalfiles.csv", NULL); + file_affected_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, + "table-affected-ovaldefs.csv", + NULL); + + if (g_file_test (file_cve, G_FILE_TEST_EXISTS) + && g_file_test (file_cpe, G_FILE_TEST_EXISTS) + && g_file_test (file_affected_products, G_FILE_TEST_EXISTS) + && g_file_test (file_ovaldefs, G_FILE_TEST_EXISTS) + && g_file_test (file_ovalfiles, G_FILE_TEST_EXISTS) + && g_file_test (file_affected_ovaldefs, G_FILE_TEST_EXISTS)) + { + sql ("COPY scap2.cves FROM '%s' WITH (FORMAT csv);", file_cve); + g_free (file_cve); + + sql ("COPY scap2.cpes FROM '%s' WITH (FORMAT csv);", file_cpe); + g_free (file_cpe); + + sql ("COPY scap2.affected_products FROM '%s' WITH (FORMAT csv);", + file_affected_products); + g_free (file_affected_products); + + sql ("COPY scap2.ovaldefs FROM '%s' WITH (FORMAT csv);", file_ovaldefs); + g_free (file_ovaldefs); + + sql ("COPY scap2.ovalfiles FROM '%s' WITH (FORMAT csv);", file_ovalfiles); + g_free (file_ovalfiles); + + sql ("COPY scap2.affected_ovaldefs FROM '%s' WITH (FORMAT csv);", + file_affected_ovaldefs); + g_free (file_affected_ovaldefs); + + /* Add the indexes, now that the data is ready. */ + + g_debug ("%s: add indexes", __func__); + proctitle_set ("gvmd: Syncing SCAP: Adding indexes"); + + if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not initialize SCAP indexes", __func__); + return -1; + } + + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + + g_info ("%s: Updating SCAP info succeeded", __func__); + proctitle_set ("gvmd: Syncing SCAP: done"); + + return 0; + } + /* Add the indexes, now that the data is ready. */ g_debug ("%s: add indexes", __func__); From b2cce6a1f68708f391b900edfe80a382bc11a2ed Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 13:23:38 +0200 Subject: [PATCH 12/25] Fix doc --- src/manage_sql_secinfo.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d37e12023..bb8f87407 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4624,11 +4624,7 @@ update_scap_timestamp () } /** - * @brief Update CERT-Bund Max CVSS. - * - * @param[in] updated_cves Whether CVEs were updated. - * @param[in] updated_cpes Whether CPEs were updated. - * @param[in] updated_ovaldefs Whether OVAL defs were updated. + * @brief Update SCAP Max CVSS. */ static void update_scap_cvss () @@ -4657,8 +4653,6 @@ update_scap_cvss () /** * @brief Update SCAP placeholder CVES. - * - * @param[in] updated_cves Whether the CVEs were updated. */ static void update_scap_placeholders () From 090a090c0e86edb47a6772b3f2d59feeb1d1bbae Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:01:41 +0200 Subject: [PATCH 13/25] Prepend new schema everywhere --- src/manage_sql_secinfo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index bb8f87407..4de5c8c29 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2651,7 +2651,7 @@ insert_cve_products (element_t list, resource_t cve, g_string_append_printf (sql_affected, "%s (%llu," - " (SELECT id FROM cpes" + " (SELECT id FROM scap2.cpes" " WHERE name='%s'))", first_affected ? "" : ",", cve, quoted_product); } @@ -3641,12 +3641,12 @@ update_ovaldef_xml (gchar **file_and_date, int private) quoted_ref_id = sql_quote (ref_id); g_free (ref_id); - sql ("INSERT INTO affected_ovaldefs (cve, ovaldef)" + sql ("INSERT INTO scap2.affected_ovaldefs (cve, ovaldef)" " SELECT cves.id, ovaldefs.id" - " FROM cves, ovaldefs" + " FROM scap2.cves, scap2.ovaldefs" " WHERE cves.name='%s'" " AND ovaldefs.name = '%s'" - " AND NOT EXISTS (SELECT * FROM affected_ovaldefs" + " AND NOT EXISTS (SELECT * FROM scap2.affected_ovaldefs" " WHERE cve = cves.id" " AND ovaldef = ovaldefs.id);", quoted_ref_id, From e830ca1fae86b6ee6e6eb9d55c625bc60786d8f1 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:02:25 +0200 Subject: [PATCH 14/25] Put scap2 first on path, otherwise scap is used --- src/manage_pg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index 2ca69959d..df1095b44 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3126,7 +3126,7 @@ manage_db_init (const gchar *name) " $$ LANGUAGE plpgsql;"); sql ("SELECT set_config ('search_path'," - " current_setting ('search_path') || ',scap2'," + " 'scap2,' || current_setting ('search_path')," " false);"); sql ("SELECT drop_scap2 ();"); From 24a147db6b3041ef3f367cdf247c5b38cae94443 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:31:38 +0200 Subject: [PATCH 15/25] Improve names --- src/manage_sql_secinfo.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 4de5c8c29..0d8608d93 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4735,7 +4735,7 @@ update_scap (gboolean reset_scap_db) return -1; } - gchar *file_cve, *file_cpe, *file_affected_products; + gchar *file_cves, *file_cpes, *file_affected_products; gchar *file_ovaldefs, *file_ovalfiles, *file_affected_ovaldefs; // GRANT pg_read_server_files TO student; @@ -4748,10 +4748,10 @@ psql -c "COPY scap.ovalfiles TO STDOUT WITH CSV" gvmd > table-ovalfiles.csv psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-ovaldefs.csv */ - file_cve = g_build_filename (GVM_SCAP_DATA_DIR, "table-cves.csv", NULL); - file_cpe = g_build_filename (GVM_SCAP_DATA_DIR, "table-cpes.csv", NULL); + file_cves = g_build_filename (GVM_SCAP_DATA_DIR, "table-cves.csv", NULL); + file_cpes = g_build_filename (GVM_SCAP_DATA_DIR, "table-cpes.csv", NULL); file_affected_products = g_build_filename (GVM_SCAP_DATA_DIR, - "table-affected.csv", + "table-affected-products.csv", NULL); file_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovaldefs.csv", NULL); file_ovalfiles = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovalfiles.csv", NULL); @@ -4759,18 +4759,18 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o "table-affected-ovaldefs.csv", NULL); - if (g_file_test (file_cve, G_FILE_TEST_EXISTS) - && g_file_test (file_cpe, G_FILE_TEST_EXISTS) + if (g_file_test (file_cves, G_FILE_TEST_EXISTS) + && g_file_test (file_cpes, G_FILE_TEST_EXISTS) && g_file_test (file_affected_products, G_FILE_TEST_EXISTS) && g_file_test (file_ovaldefs, G_FILE_TEST_EXISTS) && g_file_test (file_ovalfiles, G_FILE_TEST_EXISTS) && g_file_test (file_affected_ovaldefs, G_FILE_TEST_EXISTS)) { - sql ("COPY scap2.cves FROM '%s' WITH (FORMAT csv);", file_cve); - g_free (file_cve); + sql ("COPY scap2.cves FROM '%s' WITH (FORMAT csv);", file_cves); + g_free (file_cves); - sql ("COPY scap2.cpes FROM '%s' WITH (FORMAT csv);", file_cpe); - g_free (file_cpe); + sql ("COPY scap2.cpes FROM '%s' WITH (FORMAT csv);", file_cpes); + g_free (file_cpes); sql ("COPY scap2.affected_products FROM '%s' WITH (FORMAT csv);", file_affected_products); From 5187ea3a3a6a3e435093065e083141875087841a Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:37:15 +0200 Subject: [PATCH 16/25] Use a shared end function --- src/manage_sql_secinfo.c | 69 ++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 0d8608d93..93434c7df 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4674,6 +4674,38 @@ update_scap_placeholders () " WHERE cpes.title IS NULL;"); } +/** + * @brief Finish scap update. + * + * @return 0 success, -1 error. + */ +static int +update_scap_end () +{ + g_debug ("%s: update timestamp", __func__); + + if (update_scap_timestamp ()) + return -1; + + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + + g_info ("%s: Updating SCAP info succeeded", __func__); + proctitle_set ("gvmd: Syncing SCAP: done"); + + return 0; +} + /** * @brief Update all data in the SCAP DB. * @@ -4797,21 +4829,8 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o return -1; } - /* Replace the real scap schema with the new one. */ - - if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" - " information_schema.schemata" - " WHERE schema_name = 'scap');")) - { - sql ("ALTER SCHEMA scap RENAME TO scap3;"); - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - sql ("DROP SCHEMA scap3 CASCADE;"); - } - else - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - - g_info ("%s: Updating SCAP info succeeded", __func__); - proctitle_set ("gvmd: Syncing SCAP: done"); + if (update_scap_end ()) + goto fail; return 0; } @@ -4868,27 +4887,9 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o update_scap_placeholders (); - g_debug ("%s: update timestamp", __func__); - - if (update_scap_timestamp ()) + if (update_scap_end ()) goto fail; - /* Replace the real scap schema with the new one. */ - - if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" - " information_schema.schemata" - " WHERE schema_name = 'scap');")) - { - sql ("ALTER SCHEMA scap RENAME TO scap3;"); - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - sql ("DROP SCHEMA scap3 CASCADE;"); - } - else - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - - g_info ("%s: Updating SCAP info succeeded", __func__); - proctitle_set ("gvmd: Syncing SCAP: done"); - return 0; fail: From c27b6c7743253243a7f80b00bbeb5b56de417f68 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:39:31 +0200 Subject: [PATCH 17/25] Remove the gotos, as there's no fail code --- src/manage_sql_secinfo.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 93434c7df..ab640da24 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4829,10 +4829,7 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o return -1; } - if (update_scap_end ()) - goto fail; - - return 0; + return update_scap_end (); } /* Add the indexes, now that the data is ready. */ @@ -4856,24 +4853,24 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o proctitle_set ("gvmd: Syncing SCAP: Updating CPEs"); if (update_scap_cpes () == -1) - goto fail; + return -1; g_debug ("%s: update cves", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating CVEs"); if (update_scap_cves () == -1) - goto fail; + return -1; g_debug ("%s: update ovaldefs", __func__); proctitle_set ("gvmd: Syncing SCAP: Updating OVALdefs"); if (update_scap_ovaldefs (0 /* Feed data. */) == -1) - goto fail; + return -1; g_debug ("%s: updating user defined data", __func__); if (update_scap_ovaldefs (1 /* Private data. */) == -1) - goto fail; + return -1; /* Do calculations that need all data. */ @@ -4887,13 +4884,7 @@ psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-o update_scap_placeholders (); - if (update_scap_end ()) - goto fail; - - return 0; - - fail: - return -1; + return update_scap_end (); } /** From 2fe4688bdca32bc8de1fbfbc474b76ea024a8bed Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:45:53 +0200 Subject: [PATCH 18/25] Move CSV loading out to function --- src/manage_sql_secinfo.c | 131 ++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 63 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index ab640da24..a40e83a70 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4706,6 +4706,71 @@ update_scap_end () return 0; } +/** + * @brief Try load the feed from feed CSV files. + * + * @return 0 success, -1 error, 1 no CSV. + */ +static int +try_load_csv () +{ + gchar *file_cves, *file_cpes, *file_affected_products; + gchar *file_ovaldefs, *file_ovalfiles, *file_affected_ovaldefs; + + file_cves = g_build_filename (GVM_SCAP_DATA_DIR, "table-cves.csv", NULL); + file_cpes = g_build_filename (GVM_SCAP_DATA_DIR, "table-cpes.csv", NULL); + file_affected_products = g_build_filename (GVM_SCAP_DATA_DIR, + "table-affected-products.csv", + NULL); + file_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovaldefs.csv", NULL); + file_ovalfiles = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovalfiles.csv", NULL); + file_affected_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, + "table-affected-ovaldefs.csv", + NULL); + + if (g_file_test (file_cves, G_FILE_TEST_EXISTS) + && g_file_test (file_cpes, G_FILE_TEST_EXISTS) + && g_file_test (file_affected_products, G_FILE_TEST_EXISTS) + && g_file_test (file_ovaldefs, G_FILE_TEST_EXISTS) + && g_file_test (file_ovalfiles, G_FILE_TEST_EXISTS) + && g_file_test (file_affected_ovaldefs, G_FILE_TEST_EXISTS)) + { + sql ("COPY scap2.cves FROM '%s' WITH (FORMAT csv);", file_cves); + g_free (file_cves); + + sql ("COPY scap2.cpes FROM '%s' WITH (FORMAT csv);", file_cpes); + g_free (file_cpes); + + sql ("COPY scap2.affected_products FROM '%s' WITH (FORMAT csv);", + file_affected_products); + g_free (file_affected_products); + + sql ("COPY scap2.ovaldefs FROM '%s' WITH (FORMAT csv);", file_ovaldefs); + g_free (file_ovaldefs); + + sql ("COPY scap2.ovalfiles FROM '%s' WITH (FORMAT csv);", file_ovalfiles); + g_free (file_ovalfiles); + + sql ("COPY scap2.affected_ovaldefs FROM '%s' WITH (FORMAT csv);", + file_affected_ovaldefs); + g_free (file_affected_ovaldefs); + + /* Add the indexes, now that the data is ready. */ + + g_debug ("%s: add indexes", __func__); + proctitle_set ("gvmd: Syncing SCAP: Adding indexes"); + + if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not initialize SCAP indexes", __func__); + return -1; + } + + return update_scap_end (); + } + return 1; +} + /** * @brief Update all data in the SCAP DB. * @@ -4767,70 +4832,10 @@ update_scap (gboolean reset_scap_db) return -1; } - gchar *file_cves, *file_cpes, *file_affected_products; - gchar *file_ovaldefs, *file_ovalfiles, *file_affected_ovaldefs; - - // GRANT pg_read_server_files TO student; -/* -psql -c "COPY scap.cpes TO STDOUT WITH CSV" gvmd > table-cpes.csv -psql -c "COPY scap.cves TO STDOUT WITH CSV" gvmd > table-cves.csv -psql -c "COPY scap.affected_products TO STDOUT WITH CSV" gvmd > table-affected-products.csv -psql -c "COPY scap.ovaldefs TO STDOUT WITH CSV" gvmd > table-ovaldefs.csv -psql -c "COPY scap.ovalfiles TO STDOUT WITH CSV" gvmd > table-ovalfiles.csv -psql -c "COPY scap.affected_ovaldefs TO STDOUT WITH CSV" gvmd > table-affected-ovaldefs.csv -*/ - - file_cves = g_build_filename (GVM_SCAP_DATA_DIR, "table-cves.csv", NULL); - file_cpes = g_build_filename (GVM_SCAP_DATA_DIR, "table-cpes.csv", NULL); - file_affected_products = g_build_filename (GVM_SCAP_DATA_DIR, - "table-affected-products.csv", - NULL); - file_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovaldefs.csv", NULL); - file_ovalfiles = g_build_filename (GVM_SCAP_DATA_DIR, "table-ovalfiles.csv", NULL); - file_affected_ovaldefs = g_build_filename (GVM_SCAP_DATA_DIR, - "table-affected-ovaldefs.csv", - NULL); - - if (g_file_test (file_cves, G_FILE_TEST_EXISTS) - && g_file_test (file_cpes, G_FILE_TEST_EXISTS) - && g_file_test (file_affected_products, G_FILE_TEST_EXISTS) - && g_file_test (file_ovaldefs, G_FILE_TEST_EXISTS) - && g_file_test (file_ovalfiles, G_FILE_TEST_EXISTS) - && g_file_test (file_affected_ovaldefs, G_FILE_TEST_EXISTS)) - { - sql ("COPY scap2.cves FROM '%s' WITH (FORMAT csv);", file_cves); - g_free (file_cves); - - sql ("COPY scap2.cpes FROM '%s' WITH (FORMAT csv);", file_cpes); - g_free (file_cpes); - - sql ("COPY scap2.affected_products FROM '%s' WITH (FORMAT csv);", - file_affected_products); - g_free (file_affected_products); - - sql ("COPY scap2.ovaldefs FROM '%s' WITH (FORMAT csv);", file_ovaldefs); - g_free (file_ovaldefs); - - sql ("COPY scap2.ovalfiles FROM '%s' WITH (FORMAT csv);", file_ovalfiles); - g_free (file_ovalfiles); - - sql ("COPY scap2.affected_ovaldefs FROM '%s' WITH (FORMAT csv);", - file_affected_ovaldefs); - g_free (file_affected_ovaldefs); + /* If there's CSV in the feed, just load it. */ - /* Add the indexes, now that the data is ready. */ - - g_debug ("%s: add indexes", __func__); - proctitle_set ("gvmd: Syncing SCAP: Adding indexes"); - - if (manage_db_init_indexes ("scap")) - { - g_warning ("%s: could not initialize SCAP indexes", __func__); - return -1; - } - - return update_scap_end (); - } + if (try_load_csv () == 0) + return 0; /* Add the indexes, now that the data is ready. */ From b23e614702407a01c07859e25789ce093ffe841d Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 15:58:54 +0200 Subject: [PATCH 19/25] Remove SCAP triggers SCAP is no longer deleted. --- src/manage_pg.c | 54 ------------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index df1095b44..2b9297b51 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3206,60 +3206,6 @@ manage_db_init (const gchar *name) " FOREIGN KEY(cve) REFERENCES cves(id)," " FOREIGN KEY(ovaldef) REFERENCES ovaldefs(id));"); - /* Create deletion triggers. */ - -#if 0 - sql ("CREATE OR REPLACE FUNCTION scap_delete_affected ()" - " RETURNS TRIGGER AS $$" - " BEGIN" - " DELETE FROM affected_products where cve = old.id;" - " DELETE FROM affected_ovaldefs where cve = old.id;" - " RETURN old;" - " END;" - "$$ LANGUAGE plpgsql;"); - - sql ("CREATE TRIGGER cves_delete AFTER DELETE ON cves" - " FOR EACH ROW EXECUTE PROCEDURE scap_delete_affected ();"); - - sql ("CREATE OR REPLACE FUNCTION scap_update_cpes ()" - " RETURNS TRIGGER AS $$" - " BEGIN" - " UPDATE cpes SET max_cvss = 0.0 WHERE id = old.cpe;" - " UPDATE cpes SET cve_refs = cve_refs -1 WHERE id = old.cpe;" - " RETURN old;" - " END;" - "$$ LANGUAGE plpgsql;"); - - sql ("CREATE TRIGGER affected_delete AFTER DELETE ON affected_products" - " FOR EACH ROW EXECUTE PROCEDURE scap_update_cpes ();"); - - sql ("CREATE OR REPLACE FUNCTION scap_delete_oval ()" - " RETURNS TRIGGER AS $$" - " BEGIN" - " DELETE FROM affected_ovaldefs" - " WHERE id IN (SELECT id FROM ovaldefs" - " WHERE ovaldefs.xml_file = old.xml_file);" - " DELETE FROM ovaldefs WHERE ovaldefs.xml_file = old.xml_file;" - " RETURN old;" - " END;" - "$$ LANGUAGE plpgsql;"); - - sql ("CREATE TRIGGER ovalfiles_delete AFTER DELETE ON ovalfiles" - " FOR EACH ROW EXECUTE PROCEDURE scap_delete_oval ();"); - - sql ("CREATE OR REPLACE FUNCTION scap_update_oval ()" - " RETURNS TRIGGER AS $$" - " BEGIN" - " UPDATE ovaldefs SET max_cvss = 0.0 WHERE id = old.ovaldef;" - " RETURN old;" - " END;" - "$$ LANGUAGE plpgsql;"); - - sql ("CREATE TRIGGER affected_ovaldefs_delete" - " AFTER DELETE ON affected_ovaldefs" - " FOR EACH ROW EXECUTE PROCEDURE scap_update_oval ();"); -#endif - /* Init tables. */ sql ("INSERT INTO scap2.meta (name, value)" From 9f07794070efc3c764d8a55b8bb2a56abcdf80f8 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:16:17 +0200 Subject: [PATCH 20/25] Remove unused updated status --- src/manage_sql_secinfo.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index a40e83a70..170882772 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2949,7 +2949,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, * @param[in] xml_path XML path. * @param[in] hashed_cpes Hashed CPEs. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) @@ -2959,10 +2959,8 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) gchar *xml, *full_path; gsize xml_len; GStatBuf state; - int updated_scap_bund; int transaction_size = 0; - updated_scap_bund = 0; full_path = g_build_filename (GVM_SCAP_DATA_DIR, xml_path, NULL); if (g_stat (full_path, &state)) @@ -3015,8 +3013,6 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) if (insert_cve_from_entry (entry, last_modified, hashed_cpes, &transaction_size)) goto fail; - // FIX always true - updated_scap_bund = 1; } entry = element_next (entry); } @@ -3024,7 +3020,7 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) element_free (element); g_free (full_path); sql_commit (); - return updated_scap_bund; + return 0; fail: element_free (element); @@ -3073,17 +3069,12 @@ update_scap_cves () while ((xml_path = g_dir_read_name (dir))) if (fnmatch ("nvdcve-2.0-*.xml", xml_path, 0) == 0) { - switch (update_cve_xml (xml_path, hashed_cpes)) + if (update_cve_xml (xml_path, hashed_cpes)) { - case 0: - break; - case 1: - break; - default: - g_dir_close (dir); - g_hash_table_destroy (hashed_cpes); - cleanup_iterator (&cpes); - return -1; + g_dir_close (dir); + g_hash_table_destroy (hashed_cpes); + cleanup_iterator (&cpes); + return -1; } count++; } From 87c2600c759e313a7b105427d2207cb44a1fd466 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:19:48 +0200 Subject: [PATCH 21/25] Remove unused OVAL updated status --- src/manage_sql_secinfo.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 170882772..192869096 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3336,7 +3336,7 @@ verify_oval_file (const gchar *full_path) * @param[in] file_and_date Array containing XML path and timestamp. * @param[in] private Whether this is from the user's private dir. * - * @return 0 nothing to do, 1 updated, -1 error. + * @return 0 success, -1 error. */ static int update_ovaldef_xml (gchar **file_and_date, int private) @@ -3664,7 +3664,7 @@ update_ovaldef_xml (gchar **file_and_date, int private) g_free (quoted_xml_basename); element_free (element); sql_commit (); - return 1; + return 0; fail: g_free (quoted_xml_basename); @@ -4029,16 +4029,11 @@ update_scap_ovaldefs (int private) gchar **pair; pair = g_ptr_array_index (oval_files, index); - switch (update_ovaldef_xml (pair, private)) + if (update_ovaldef_xml (pair, private)) { - case 0: - break; - case 1: - break; - default: - oval_files_free (); - g_free (oval_dir); - return -1; + oval_files_free (); + g_free (oval_dir); + return -1; } count++; } From b9269921970db2eeadba95d12e5f37340b459312 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:26:24 +0200 Subject: [PATCH 22/25] Remove old case that was for date check --- src/manage_sql_secinfo.c | 357 ++++++++++++++++++--------------------- 1 file changed, 169 insertions(+), 188 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 192869096..129715658 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3439,6 +3439,12 @@ update_ovaldef_xml (gchar **file_and_date, int private) { int definition_date_newest, definition_date_oldest; gchar *quoted_id, *quoted_oval_id; + element_t metadata, title, description, repository, reference; + element_t status; + gchar *deprecated, *version, *id, *id_value, *class; + gchar *quoted_title, *quoted_class, *quoted_description; + gchar *quoted_status, *status_text; + int cve_count; /* The newest and oldest of this definition's dates (created, * modified, etc), from the OVAL XML. */ @@ -3446,213 +3452,188 @@ update_ovaldef_xml (gchar **file_and_date, int private) &definition_date_newest, &definition_date_oldest); - if (0) + id_value = element_attribute (definition, "id"); + if (id_value == NULL) { - gchar *id; - - // FIX remove. now always full update - - id = element_attribute (definition, "id"); - quoted_oval_id = sql_quote (id ? id : ""); - g_free (id); - g_info ("%s: Filtered %s (%i)", - __func__, - quoted_oval_id, - definition_date_oldest); - g_free (quoted_oval_id); + g_warning ("%s: oval_definition missing id", + __func__); + element_free (element); + goto fail; } - else - { - element_t metadata, title, description, repository, reference; - element_t status; - gchar *deprecated, *version, *id, *id_value, *class; - gchar *quoted_title, *quoted_class, *quoted_description; - gchar *quoted_status, *status_text; - int cve_count; - - id_value = element_attribute (definition, "id"); - if (id_value == NULL) - { - g_warning ("%s: oval_definition missing id", - __func__); - element_free (element); - goto fail; - } - metadata = element_child (definition, "metadata"); - if (metadata == NULL) - { - g_warning ("%s: metadata missing", - __func__); - element_free (element); - g_free (id_value); - goto fail; - } + metadata = element_child (definition, "metadata"); + if (metadata == NULL) + { + g_warning ("%s: metadata missing", + __func__); + element_free (element); + g_free (id_value); + goto fail; + } - title = element_child (metadata, "title"); - if (title == NULL) - { - g_warning ("%s: title missing", - __func__); - element_free (element); - g_free (id_value); - goto fail; - } + title = element_child (metadata, "title"); + if (title == NULL) + { + g_warning ("%s: title missing", + __func__); + element_free (element); + g_free (id_value); + goto fail; + } - description = element_child (metadata, "description"); - if (description == NULL) - { - g_warning ("%s: description missing", - __func__); - element_free (element); - g_free (id_value); - goto fail; - } + description = element_child (metadata, "description"); + if (description == NULL) + { + g_warning ("%s: description missing", + __func__); + element_free (element); + g_free (id_value); + goto fail; + } - repository = element_child (metadata, "oval_repository"); - if (repository == NULL) - { - g_warning ("%s: oval_repository missing", - __func__); - element_free (element); - g_free (id_value); - goto fail; - } + repository = element_child (metadata, "oval_repository"); + if (repository == NULL) + { + g_warning ("%s: oval_repository missing", + __func__); + element_free (element); + g_free (id_value); + goto fail; + } - cve_count = 0; - reference = element_first_child (metadata); - while (reference) + cve_count = 0; + reference = element_first_child (metadata); + while (reference) + { + if (strcmp (element_name (reference), "reference") == 0) { - if (strcmp (element_name (reference), "reference") == 0) - { - gchar *source; + gchar *source; - source = element_attribute (reference, "source"); - if (source && strcasecmp (source, "cve") == 0) - cve_count++; - g_free (source); - } - reference = element_next (reference); + source = element_attribute (reference, "source"); + if (source && strcasecmp (source, "cve") == 0) + cve_count++; + g_free (source); } + reference = element_next (reference); + } - id = g_strdup_printf ("%s_%s", id_value, xml_basename); - quoted_id = sql_quote (id); - g_free (id); - quoted_oval_id = sql_quote (id_value); - g_free (id_value); - - version = element_attribute (definition, "version"); - if (g_regex_match_simple ("^[0-9]+$", (gchar *) version, 0, 0) == 0) - { - g_warning ("%s: invalid version: %s", - __func__, - version); - element_free (element); - g_free (version); - goto fail; - } + id = g_strdup_printf ("%s_%s", id_value, xml_basename); + quoted_id = sql_quote (id); + g_free (id); + quoted_oval_id = sql_quote (id_value); + g_free (id_value); - class = element_attribute (definition, "class"); - quoted_class = sql_quote (class); - g_free (class); - quoted_title = sql_quote_element_text (title); - quoted_description = sql_quote_element_text (description); - status = element_child (repository, "status"); - deprecated = element_attribute (definition, "deprecated"); - status_text = NULL; - if (status) - status_text = element_text (status); - if (status_text && strlen (status_text)) - quoted_status = sql_quote (status_text); - else if (deprecated && strcasecmp (deprecated, "TRUE")) - quoted_status = sql_quote ("DEPRECATED"); - else - quoted_status = sql_quote (""); - g_free (status_text); - - sql ("INSERT INTO scap2.ovaldefs" - " (uuid, name, comment, creation_time," - " modification_time, version, deprecated, def_class," - " title, description, xml_file, status," - " max_cvss, cve_refs)" - " VALUES ('%s', '%s', '', %i, %i, %s, %i, '%s', '%s'," - " '%s', '%s', '%s', 0.0, %i)" - " ON CONFLICT (uuid) DO UPDATE" - " SET name = EXCLUDED.name," - " comment = EXCLUDED.comment," - " creation_time = EXCLUDED.creation_time," - " modification_time = EXCLUDED.modification_time," - " version = EXCLUDED.version," - " deprecated = EXCLUDED.deprecated," - " def_class = EXCLUDED.def_class," - " title = EXCLUDED.title," - " description = EXCLUDED.description," - " xml_file = EXCLUDED.xml_file," - " status = EXCLUDED.status," - " max_cvss = 0.0," - " cve_refs = EXCLUDED.cve_refs;", - quoted_id, - quoted_oval_id, - definition_date_oldest == 0 - ? file_timestamp - : definition_date_newest, - definition_date_oldest == 0 - ? file_timestamp - : definition_date_oldest, - version, - (deprecated && strcasecmp (deprecated, "TRUE")) ? 1 : 0, - quoted_class, - quoted_title, - quoted_description, - quoted_xml_basename, - quoted_status, - cve_count); - increment_transaction_size (&transaction_size); - g_free (quoted_id); - g_free (quoted_class); - g_free (quoted_title); - g_free (quoted_description); - g_free (quoted_status); - g_free (deprecated); + version = element_attribute (definition, "version"); + if (g_regex_match_simple ("^[0-9]+$", (gchar *) version, 0, 0) == 0) + { + g_warning ("%s: invalid version: %s", + __func__, + version); + element_free (element); g_free (version); + goto fail; + } + + class = element_attribute (definition, "class"); + quoted_class = sql_quote (class); + g_free (class); + quoted_title = sql_quote_element_text (title); + quoted_description = sql_quote_element_text (description); + status = element_child (repository, "status"); + deprecated = element_attribute (definition, "deprecated"); + status_text = NULL; + if (status) + status_text = element_text (status); + if (status_text && strlen (status_text)) + quoted_status = sql_quote (status_text); + else if (deprecated && strcasecmp (deprecated, "TRUE")) + quoted_status = sql_quote ("DEPRECATED"); + else + quoted_status = sql_quote (""); + g_free (status_text); + + sql ("INSERT INTO scap2.ovaldefs" + " (uuid, name, comment, creation_time," + " modification_time, version, deprecated, def_class," + " title, description, xml_file, status," + " max_cvss, cve_refs)" + " VALUES ('%s', '%s', '', %i, %i, %s, %i, '%s', '%s'," + " '%s', '%s', '%s', 0.0, %i)" + " ON CONFLICT (uuid) DO UPDATE" + " SET name = EXCLUDED.name," + " comment = EXCLUDED.comment," + " creation_time = EXCLUDED.creation_time," + " modification_time = EXCLUDED.modification_time," + " version = EXCLUDED.version," + " deprecated = EXCLUDED.deprecated," + " def_class = EXCLUDED.def_class," + " title = EXCLUDED.title," + " description = EXCLUDED.description," + " xml_file = EXCLUDED.xml_file," + " status = EXCLUDED.status," + " max_cvss = 0.0," + " cve_refs = EXCLUDED.cve_refs;", + quoted_id, + quoted_oval_id, + definition_date_oldest == 0 + ? file_timestamp + : definition_date_newest, + definition_date_oldest == 0 + ? file_timestamp + : definition_date_oldest, + version, + (deprecated && strcasecmp (deprecated, "TRUE")) ? 1 : 0, + quoted_class, + quoted_title, + quoted_description, + quoted_xml_basename, + quoted_status, + cve_count); + increment_transaction_size (&transaction_size); + g_free (quoted_id); + g_free (quoted_class); + g_free (quoted_title); + g_free (quoted_description); + g_free (quoted_status); + g_free (deprecated); + g_free (version); - reference = element_first_child (metadata); - while (reference) + reference = element_first_child (metadata); + while (reference) + { + if (strcmp (element_name (reference), "reference") == 0) { - if (strcmp (element_name (reference), "reference") == 0) - { - gchar *source; + gchar *source; - source = element_attribute (reference, "source"); - if (source && strcasecmp (source, "cve") == 0) - { - gchar *ref_id, *quoted_ref_id; - - ref_id = element_attribute (reference, "ref_id"); - quoted_ref_id = sql_quote (ref_id); - g_free (ref_id); - - sql ("INSERT INTO scap2.affected_ovaldefs (cve, ovaldef)" - " SELECT cves.id, ovaldefs.id" - " FROM scap2.cves, scap2.ovaldefs" - " WHERE cves.name='%s'" - " AND ovaldefs.name = '%s'" - " AND NOT EXISTS (SELECT * FROM scap2.affected_ovaldefs" - " WHERE cve = cves.id" - " AND ovaldef = ovaldefs.id);", - quoted_ref_id, - quoted_oval_id); - - g_free (quoted_ref_id); - increment_transaction_size (&transaction_size); - } - g_free (source); + source = element_attribute (reference, "source"); + if (source && strcasecmp (source, "cve") == 0) + { + gchar *ref_id, *quoted_ref_id; + + ref_id = element_attribute (reference, "ref_id"); + quoted_ref_id = sql_quote (ref_id); + g_free (ref_id); + + sql ("INSERT INTO scap2.affected_ovaldefs (cve, ovaldef)" + " SELECT cves.id, ovaldefs.id" + " FROM scap2.cves, scap2.ovaldefs" + " WHERE cves.name='%s'" + " AND ovaldefs.name = '%s'" + " AND NOT EXISTS (SELECT * FROM scap2.affected_ovaldefs" + " WHERE cve = cves.id" + " AND ovaldef = ovaldefs.id);", + quoted_ref_id, + quoted_oval_id); + + g_free (quoted_ref_id); + increment_transaction_size (&transaction_size); } - reference = element_next (reference); + g_free (source); } - - g_free (quoted_oval_id); + reference = element_next (reference); } + + g_free (quoted_oval_id); } definition = element_next (definition); } From 2a8872ff86dc2d67c9019e746c15b91411f7f973 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:35:22 +0200 Subject: [PATCH 23/25] Remove note, migration is still required --- src/manage_sql_secinfo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 129715658..d90059e65 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4504,7 +4504,6 @@ manage_sync_cert (sigset_t *sigmask_current) int check_scap_db_version () { - // FIX drop with full rebuild switch (manage_scap_db_version ()) { /* TODO The sync script had a whole lot of migrators in here. */ From 15568591020fe1f618238c849d135aaa5f8e8f2e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:42:04 +0200 Subject: [PATCH 24/25] Remove note, still required --- src/manage_sql_secinfo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d90059e65..1c21d0600 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4077,7 +4077,6 @@ update_scap_ovaldefs (int private) } cleanup_iterator (&files); - // FIX possibly can remove too sql ("DELETE FROM scap2.ovaldefs" " WHERE (xml_file NOT LIKE 'oval/%%')" "%s;", From 5d8aa405147ff82cf4eb171c9a0f7e421d21939e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 1 Jun 2020 16:43:19 +0200 Subject: [PATCH 25/25] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aba093f1..5b40a5055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Simplify sync lockfile handling [#1059](https://github.com/greenbone/gvmd/pull/1059) - Do not ignore empty hosts_allow and ifaces_allow [#1064](https://github.com/greenbone/gvmd/pull/1064) - Reduce the memory cache of NVTs [#1076](https://github.com/greenbone/gvmd/pull/1076) +- Sync SCAP using a second schema [#1111](https://github.com/greenbone/gvmd/pull/1111) ### Fixed - Add NULL check in nvts_feed_version_epoch [#768](https://github.com/greenbone/gvmd/pull/768)