Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle removed CPEs and CVEs in SCAP sync #1097

Merged
merged 7 commits into from
May 20, 2020
Merged

Handle removed CPEs and CVEs in SCAP sync #1097

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2879642
Remove no longer affected products when updating CVE
mattmundell May 18, 2020
4b55384
In SCAP sync remove CVEs that were removed from XML
mattmundell May 19, 2020
6cd23f3
In SCAP sync remove CPEs that were removed from XML
mattmundell May 19, 2020
74568c9
Free id
mattmundell May 19, 2020
6fa9374
Mark SCAP updated flags when deleting
mattmundell May 19, 2020
3c71843
Update changelog
mattmundell May 19, 2020
980f349
Merge branch 'master' into cpe-removal
mattmundell May 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Fix failure detection for xml_split command [#1074](https://github.com/greenbone/gvmd/pull/1074)
- Fix deletion of OVAL definition data [#1079](https://github.com/greenbone/gvmd/pull/1079)
- Fix feed lock in sync script [#1088](https://github.com/greenbone/gvmd/pull/1088)
- Handle removed CPEs and CVEs in SCAP sync [#1097](https://github.com/greenbone/gvmd/pull/1097)

### Removed
- Remove support for "All SecInfo": removal of "allinfo" for type in get_info [#790](https://github.com/greenbone/gvmd/pull/790)
Expand Down
164 changes: 153 additions & 11 deletions src/manage_sql_secinfo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2329,11 +2329,13 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata
*
* @param[in] path Path to file.
* @param[in] last_cve_update Time of last CVE update.
* @param[in] all_xml_cpes String to add all CPEs to.
*
* @return 0 nothing to do, 1 updated, -1 error.
*/
static int
update_scap_cpes_from_file (const gchar *path, int last_cve_update)
update_scap_cpes_from_file (const gchar *path, int last_cve_update,
GString *all_xml_cpes)
{
GError *error;
element_t element, cpe_list, cpe_item;
Expand Down Expand Up @@ -2393,7 +2395,7 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update)
cpe_item = element_first_child (cpe_list);
while (cpe_item)
{
gchar *modification_date;
gchar *modification_date, *name;
int modification_time;
element_t item_metadata;

Expand Down Expand Up @@ -2421,6 +2423,19 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update)
modification_time = parse_iso_time (modification_date);
g_free (modification_date);

name = element_attribute (cpe_item, "name");
if (name == NULL)
{
g_warning ("%s: name missing", __func__);
goto fail;
}

if (all_xml_cpes->str[0] == '\0')
g_string_append_printf (all_xml_cpes, "'%s'", name);
else
g_string_append_printf (all_xml_cpes, ",'%s'", name);
g_free (name);

if (modification_time > last_cve_update)
{
if (insert_scap_cpe (&inserts, cpe_item, item_metadata,
Expand Down Expand Up @@ -2451,11 +2466,12 @@ update_scap_cpes_from_file (const gchar *path, int last_cve_update)
* @brief Update SCAP CPEs.
*
* @param[in] last_scap_update Time of last SCAP update.
* @param[in] all_xml_cpes String to add all CPEs to.
*
* @return 0 nothing to do, 1 updated, -1 error.
*/
static int
update_scap_cpes (int last_scap_update)
update_scap_cpes (int last_scap_update, GString *all_xml_cpes)
{
gchar *full_path;
const gchar *split_dir;
Expand Down Expand Up @@ -2495,7 +2511,8 @@ update_scap_cpes (int last_scap_update)
g_warning ("%s: Failed to split CPEs, attempting with full file",
__func__);
updated_scap_cpes = update_scap_cpes_from_file (full_path,
last_cve_update);
last_cve_update,
all_xml_cpes);
g_free (full_path);
return updated_scap_cpes;
}
Expand All @@ -2516,7 +2533,7 @@ update_scap_cpes (int last_scap_update)
break;
}

ret = update_scap_cpes_from_file (path, last_cve_update);
ret = update_scap_cpes_from_file (path, last_cve_update, all_xml_cpes);
g_free (path);
if (ret < 0)
{
Expand Down Expand Up @@ -2580,6 +2597,19 @@ hashed_cpes_cpe_id (GHashTable *hashed_cpes, const gchar *product_tilde)
return GPOINTER_TO_INT (g_hash_table_lookup (hashed_cpes, product_tilde));
}

/**
* @brief Push a generic pointer onto an array.
*
* @param[in] array Array.
* @param[in] pointer Pointer.
*/
static void
array_remove (array_t *array, gpointer pointer)
{
if (array)
g_ptr_array_remove_fast (array, pointer);
}

/**
* @brief Insert products for a CVE.
*
Expand All @@ -2598,6 +2628,8 @@ insert_cve_products (element_t list, resource_t cve,
element_t product;
int first_product, first_affected;
GString *sql_cpes, *sql_affected;
array_t *initial_affected;
iterator_t rows;

if (list == NULL)
return;
Expand All @@ -2615,6 +2647,17 @@ insert_cve_products (element_t list, resource_t cve,
" (cve, cpe)"
" VALUES");

/* Record existing affected products. */

initial_affected = make_array ();
init_iterator (&rows,
"SELECT cpe FROM scap.affected_products"
" WHERE cve = %llu;",
cve);
while (next (&rows))
array_add (initial_affected, GINT_TO_POINTER (iterator_int64 (&rows, 0)));
cleanup_iterator (&rows);

/* Buffer the SQL. */

first_product = first_affected = 1;
Expand Down Expand Up @@ -2679,15 +2722,22 @@ insert_cve_products (element_t list, resource_t cve,
}
else
{
int cpe;

/* The product is in the db.
*
* So we don't need to insert it. */

cpe = hashed_cpes_cpe_id (hashed_cpes, product_tilde);

g_string_append_printf
(sql_affected,
"%s (%llu, %i)",
first_affected ? "" : ",", cve,
hashed_cpes_cpe_id (hashed_cpes, product_tilde));
cpe);

/* Remove the affected product from initial_affected. */
array_remove (initial_affected, GINT_TO_POINTER (cpe));
}

first_affected = 0;
Expand All @@ -2700,6 +2750,27 @@ insert_cve_products (element_t list, resource_t cve,
product = element_next (product);
}

/* Remove affected_products that remain in initial_affected. */

if (initial_affected->len)
{
guint index;
GString *ids;

ids = g_string_new ("");
for (index = 0; index < initial_affected->len; index++)
g_string_append_printf (ids,
"%s%i",
index == 0 ? "" : ",",
GPOINTER_TO_INT (g_ptr_array_index
(initial_affected, index)));
sql ("DELETE from scap.affected_products"
" WHERE cve = %llu AND cpe = ANY(array[%s]);",
cve, ids->str);
g_string_free (ids, TRUE);
}
g_ptr_array_free (initial_affected, TRUE);

/* Run the SQL. */

if (first_product == 0)
Expand Down Expand Up @@ -2968,12 +3039,14 @@ insert_cve_from_entry (element_t entry, element_t last_modified,
* @param[in] last_scap_update Time of last SCAP update.
* @param[in] last_cve_update Time of last update to a DFN.
* @param[in] hashed_cpes Hashed CPEs.
* @param[in] all_xml_cves String to add all CVEs to.
*
* @return 0 nothing to do, 1 updated, -1 error.
*/
static int
update_cve_xml (const gchar *xml_path, int last_scap_update,
int last_cve_update, GHashTable *hashed_cpes)
int last_cve_update, GHashTable *hashed_cpes,
GString *all_xml_cves)
{
GError *error;
element_t element, entry;
Expand Down Expand Up @@ -3032,6 +3105,7 @@ update_cve_xml (const gchar *xml_path, int last_scap_update,
{
if (strcmp (element_name (entry), "entry") == 0)
{
gchar *id;
element_t last_modified;

last_modified = element_child (entry, "vuln:last-modified-datetime");
Expand All @@ -3041,6 +3115,21 @@ update_cve_xml (const gchar *xml_path, int last_scap_update,
__func__);
goto fail;
}

id = element_attribute (entry, "id");
if (id == NULL)
{
g_warning ("%s: id missing",
__func__);
goto fail;
}

if (all_xml_cves->str[0] == '\0')
g_string_append_printf (all_xml_cves, "'%s'", id);
else
g_string_append_printf (all_xml_cves, ",'%s'", id);
g_free (id);

if (parse_iso_time_element_text (last_modified) > last_cve_update)
{
if (insert_cve_from_entry (entry, last_modified, hashed_cpes,
Expand Down Expand Up @@ -3084,6 +3173,7 @@ update_scap_cves (int last_scap_update)
const gchar *xml_path;
GHashTable *hashed_cpes;
iterator_t cpes;
GString *all_xml_cves;

error = NULL;
dir = g_dir_open (GVM_SCAP_DATA_DIR, 0, &error);
Expand All @@ -3105,13 +3195,15 @@ update_scap_cves (int last_scap_update)
(gpointer*) iterator_string (&cpes, 0),
GINT_TO_POINTER (iterator_int (&cpes, 1)));

all_xml_cves = g_string_new ("");

count = 0;
updated_scap_cves = 0;
while ((xml_path = g_dir_read_name (dir)))
if (fnmatch ("nvdcve-2.0-*.xml", xml_path, 0) == 0)
{
switch (update_cve_xml (xml_path, last_scap_update, last_cve_update,
hashed_cpes))
hashed_cpes, all_xml_cves))
{
case 0:
break;
Expand All @@ -3121,12 +3213,35 @@ update_scap_cves (int last_scap_update)
default:
g_dir_close (dir);
g_hash_table_destroy (hashed_cpes);
g_string_free (all_xml_cves, TRUE);
cleanup_iterator (&cpes);
return -1;
}
count++;
}

if (strlen (all_xml_cves->str) > 0)
{
/* Remove CVES from the db that are not in all_xml_cves. */

g_string_prepend
(all_xml_cves, "WITH"
" removed AS (SELECT id FROM scap.cves"
" WHERE NOT uuid = ANY(array[");

g_string_append
(all_xml_cves, "])),"
" dummy AS (DELETE FROM scap.affected_products"
" WHERE cve IN (SELECT id FROM removed)"
" RETURNING *)"
" DELETE FROM scap.cves"
" WHERE id IN (SELECT id FROM removed);");

sql (all_xml_cves->str);
updated_scap_cves = 1;
}
g_string_free (all_xml_cves, TRUE);

if (count == 0)
g_warning ("No CVEs found in %s", GVM_SCAP_DATA_DIR);

Expand Down Expand Up @@ -4817,6 +4932,7 @@ update_scap (gboolean ignore_last_scap_update,
{
int last_feed_update, last_scap_update;
int updated_scap_ovaldefs, updated_scap_cpes, updated_scap_cves;
GString *all_xml_cpes;

updated_scap_ovaldefs = 0;
updated_scap_cpes = 0;
Expand Down Expand Up @@ -4884,14 +5000,19 @@ update_scap (gboolean ignore_last_scap_update,

g_info ("%s: Updating data from feed", __func__);

all_xml_cpes = g_string_new ("");

if (update_cpes)
{
g_debug ("%s: update cpes", __func__);
proctitle_set ("gvmd: Syncing SCAP: Updating CPEs");

updated_scap_cpes = update_scap_cpes (last_scap_update);
updated_scap_cpes = update_scap_cpes (last_scap_update, all_xml_cpes);
if (updated_scap_cpes == -1)
goto fail;
{
g_string_free (all_xml_cpes, TRUE);
goto fail;
}
}

if (update_cves)
Expand All @@ -4901,8 +5022,29 @@ update_scap (gboolean ignore_last_scap_update,

updated_scap_cves = update_scap_cves (last_scap_update);
if (updated_scap_cves == -1)
goto fail;
{
g_string_free (all_xml_cpes, TRUE);
goto fail;
}
}

if (strlen (all_xml_cpes->str) > 0)
{
/* Remove CPES not in all_xml_cpes, except those in affected_products. */

g_string_prepend
(all_xml_cpes, "DELETE FROM scap.cpes"
" WHERE NOT EXISTS (SELECT * FROM scap.affected_products"
" WHERE cpe = scap.cpes.id)"
" AND NOT uuid = ANY(array[");

g_string_append
(all_xml_cpes, "]);");

sql (all_xml_cpes->str);
updated_scap_cpes = 1;
}
g_string_free (all_xml_cpes, TRUE);

if (update_ovaldefs)
{
Expand Down