From 69710f6171f0d78357e87112bbcf4bee03bceca0 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 17 Apr 2023 16:22:44 +0200 Subject: [PATCH 1/3] Rebuild NVTs in shadow tables --- src/manage.h | 2 +- src/manage_pg.c | 101 ++++++++++++++++++++++----------------- src/manage_sql_configs.c | 22 +++++---- src/manage_sql_nvts.c | 68 +++++++++++++++++++------- 4 files changed, 121 insertions(+), 72 deletions(-) diff --git a/src/manage.h b/src/manage.h index 20f7954d7..578c536e4 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1919,7 +1919,7 @@ nvt_selector_iterator_type (iterator_t*); /* NVT preferences. */ void -manage_nvt_preference_add (const char*, const char*); +manage_nvt_preference_add (const char*, const char*, int); void manage_nvt_preferences_enable (); diff --git a/src/manage_pg.c b/src/manage_pg.c index 03b130f4e..6acb4a903 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -1747,6 +1747,62 @@ create_view_vulns () #undef VULNS_RESULTS_WHERE +/** + * @brief Create NVT related tables. + */ +void +create_tables_nvt (const gchar *suffix) +{ + sql ("CREATE TABLE IF NOT EXISTS vt_refs%s" + " (id SERIAL PRIMARY KEY," + " vt_oid text NOT NULL," + " type text NOT NULL," + " ref_id text NOT NULL," + " ref_text text);", + suffix); + + sql ("CREATE TABLE IF NOT EXISTS vt_severities%s" + " (id SERIAL PRIMARY KEY," + " vt_oid text NOT NULL," + " type text NOT NULL," + " origin text," + " date integer," + " score double precision," + " value text);", + suffix); + + sql ("CREATE TABLE IF NOT EXISTS nvt_preferences%s" + " (id SERIAL PRIMARY KEY," + " name text UNIQUE NOT NULL," + " value text);", + suffix); + + sql ("CREATE TABLE IF NOT EXISTS nvts%s" + " (id SERIAL PRIMARY KEY," + " uuid text UNIQUE NOT NULL," + " oid text UNIQUE NOT NULL," + " name text," + " comment text," + " summary text," + " insight text," + " affected text," + " impact text," + " cve text," + " tag text," + " category text," + " family text," + " cvss_base text," + " creation_time integer," + " modification_time integer," + " solution text," + " solution_type text," + " solution_method text," + " detection text," + " qod integer," + " qod_type text);", + suffix); +} + /** * @brief Create all tables. */ @@ -2565,50 +2621,7 @@ create_tables () " name text," " value text);"); - sql ("CREATE TABLE IF NOT EXISTS vt_refs" - " (id SERIAL PRIMARY KEY," - " vt_oid text NOT NULL," - " type text NOT NULL," - " ref_id text NOT NULL," - " ref_text text);"); - - sql ("CREATE TABLE IF NOT EXISTS vt_severities" - " (id SERIAL PRIMARY KEY," - " vt_oid text NOT NULL," - " type text NOT NULL," - " origin text," - " date integer," - " score double precision," - " value text);"); - - sql ("CREATE TABLE IF NOT EXISTS nvt_preferences" - " (id SERIAL PRIMARY KEY," - " name text UNIQUE NOT NULL," - " value text);"); - - sql ("CREATE TABLE IF NOT EXISTS nvts" - " (id SERIAL PRIMARY KEY," - " uuid text UNIQUE NOT NULL," - " oid text UNIQUE NOT NULL," - " name text," - " comment text," - " summary text," - " insight text," - " affected text," - " impact text," - " cve text," - " tag text," - " category text," - " family text," - " cvss_base text," - " creation_time integer," - " modification_time integer," - " solution text," - " solution_type text," - " solution_method text," - " detection text," - " qod integer," - " qod_type text);"); + create_tables_nvt (""); sql ("CREATE TABLE IF NOT EXISTS notes" " (id SERIAL PRIMARY KEY," diff --git a/src/manage_sql_configs.c b/src/manage_sql_configs.c index 5e3ff59d1..abc63a844 100644 --- a/src/manage_sql_configs.c +++ b/src/manage_sql_configs.c @@ -1694,25 +1694,29 @@ check_config_families () /** * @brief Add/replace an NVT preference. * - * @param[in] name The name of the preference. - * @param[in] value The value of the preference. + * @param[in] name The name of the preference. + * @param[in] value The value of the preference. + * @param[in] rebuild Whether a rebuild is happening. */ void -manage_nvt_preference_add (const char* name, const char* value) +manage_nvt_preference_add (const char* name, const char* value, int rebuild) { gchar* quoted_name = sql_quote (name); gchar* quoted_value = sql_quote (value); if (strcmp (name, "port_range")) { - if (sql_int ("SELECT EXISTS" - " (SELECT * FROM nvt_preferences" - " WHERE name = '%s')", - quoted_name)) - sql ("DELETE FROM nvt_preferences WHERE name = '%s';", quoted_name); + if (rebuild == 0) { + if (sql_int ("SELECT EXISTS" + " (SELECT * FROM nvt_preferences" + " WHERE name = '%s')", + quoted_name)) + sql ("DELETE FROM nvt_preferences WHERE name = '%s';", quoted_name); + } - sql ("INSERT into nvt_preferences (name, value)" + sql ("INSERT into nvt_preferences%s (name, value)" " VALUES ('%s', '%s');", + rebuild ? "_rebuild" : "", quoted_name, quoted_value); } diff --git a/src/manage_sql_nvts.c b/src/manage_sql_nvts.c index 13d5381f3..26ce909b2 100644 --- a/src/manage_sql_nvts.c +++ b/src/manage_sql_nvts.c @@ -50,6 +50,12 @@ */ #define G_LOG_DOMAIN "md manage" + +/* Headers from backend specific manage_xxx.c file. */ + +void +create_tables_nvt (const gchar *); + /* NVT related global options */ @@ -269,7 +275,9 @@ insert_vt_refs (const nvti_t *nvti, int truncate) int i; if (truncate == 0) - sql ("DELETE FROM vt_refs where vt_oid = '%s';", nvti_oid (nvti)); + sql ("DELETE FROM vt_refs%s where vt_oid = '%s';", + truncate ? "_rebuild" : "", + nvti_oid (nvti)); for (i = 0; i < nvti_vtref_len (nvti); i++) { @@ -281,8 +289,9 @@ insert_vt_refs (const nvti_t *nvti, int truncate) quoted_id = sql_quote (vtref_id (ref)); quoted_text = sql_quote (vtref_text (ref) ? vtref_text (ref) : ""); - sql ("INSERT into vt_refs (vt_oid, type, ref_id, ref_text)" + sql ("INSERT into vt_refs%s (vt_oid, type, ref_id, ref_text)" " VALUES ('%s', '%s', '%s', '%s');", + truncate ? "_rebuild" : "", nvti_oid (nvti), quoted_type, quoted_id, quoted_text); g_free (quoted_type); @@ -306,7 +315,9 @@ insert_vt_severities (const nvti_t *nvti, int truncate) double highest; if (truncate == 0) - sql ("DELETE FROM vt_severities where vt_oid = '%s';", nvti_oid (nvti)); + sql ("DELETE FROM vt_severities%s where vt_oid = '%s';", + truncate ? "_rebuild" : "", + nvti_oid (nvti)); highest = 0; @@ -321,9 +332,10 @@ insert_vt_severities (const nvti_t *nvti, int truncate) quoted_value = sql_quote (vtseverity_value (severity) ? vtseverity_value (severity) : ""); - sql ("INSERT into vt_severities (vt_oid, type, origin, date, score," - " value)" + sql ("INSERT into vt_severities%s (vt_oid, type, origin, date, score," + " value)" " VALUES ('%s', '%s', '%s', %i, %0.1f, '%s');", + truncate ? "_rebuild" : "", nvti_oid (nvti), vtseverity_type (severity), quoted_origin, vtseverity_date (severity), vtseverity_score (severity), quoted_value); @@ -392,18 +404,21 @@ insert_nvt (const nvti_t *nvti, int truncate) if ((truncate == 0) && sql_int ("SELECT EXISTS (SELECT * FROM nvts WHERE oid = '%s');", nvti_oid (nvti))) - sql ("DELETE FROM nvts WHERE oid = '%s';", nvti_oid (nvti)); + sql ("DELETE FROM nvts%s WHERE oid = '%s';", + truncate ? "_rebuild" : "", + nvti_oid (nvti)); insert_vt_refs(nvti, truncate); highest = insert_vt_severities(nvti, truncate); - sql ("INSERT into nvts (oid, name, summary, insight, affected," + sql ("INSERT into nvts%s (oid, name, summary, insight, affected," " impact, cve, tag, category, family, cvss_base," " creation_time, modification_time, uuid, solution_type," " solution_method, solution, detection, qod, qod_type)" " VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s'," " '%s', %i, '%s', %0.1f, %i, %i, '%s', '%s', '%s', '%s', '%s', %d, '%s');", + truncate ? "_rebuild" : "", nvti_oid (nvti), quoted_name, quoted_summary, quoted_insight, quoted_affected, quoted_impact, quoted_cve, quoted_tag, nvti_category (nvti), quoted_family, highest, @@ -1120,11 +1135,10 @@ family_count () * @brief Insert a NVT preferences. * * @param[in] nvt_preference Preference. - * @param[in] dummy Dummy arg for g_list_foreach. - * + * @param[in] rebuild Whether a rebuild is happening. */ static void -insert_nvt_preference (gpointer nvt_preference, gpointer dummy) +insert_nvt_preference (gpointer nvt_preference, gpointer rebuild) { preference_t *preference; @@ -1132,18 +1146,19 @@ insert_nvt_preference (gpointer nvt_preference, gpointer dummy) return; preference = (preference_t*) nvt_preference; - manage_nvt_preference_add (preference->name, preference->value); + manage_nvt_preference_add (preference->name, preference->value, GPOINTER_TO_INT (rebuild)); } /** * @brief Inserts NVT preferences in DB from a list of nvt_preference_t structures. * - * @param[in] nvt_preferences_list List of nvts to be inserted. + * @param[in] nvt_preferences_list List of nvts to be inserted. + * @param[in] rebuild Whether a rebuild is happening. */ static void -insert_nvt_preferences_list (GList *nvt_preferences_list) +insert_nvt_preferences_list (GList *nvt_preferences_list, int rebuild) { - g_list_foreach (nvt_preferences_list, insert_nvt_preference, NULL); + g_list_foreach (nvt_preferences_list, insert_nvt_preference, GINT_TO_POINTER (rebuild)); } /** @@ -1547,8 +1562,12 @@ update_nvts_from_vts (entity_t *get_vts_response, sql_begin_immediate (); if (truncate) { - sql ("TRUNCATE nvts;"); - sql ("TRUNCATE nvt_preferences;"); + sql ("DROP TABLE IF EXISTS vt_refs_rebuild;"); + sql ("DROP TABLE IF EXISTS vt_severities_rebuild;"); + sql ("DROP TABLE IF EXISTS nvt_preferences_rebuild;"); + sql ("DROP TABLE IF EXISTS nvts_rebuild;"); + + create_tables_nvt ("_rebuild"); } else if (sql_int ("SELECT coalesce ((SELECT CAST (value AS INTEGER)" " FROM meta" @@ -1591,15 +1610,28 @@ update_nvts_from_vts (entity_t *get_vts_response, return -1; } if (truncate == 0) - sql ("DELETE FROM nvt_preferences WHERE name LIKE '%s:%%';", + sql ("DELETE FROM nvt_preferences%s WHERE name LIKE '%s:%%';", + truncate ? "_rebuild" : "", nvti_oid (nvti)); - insert_nvt_preferences_list (preferences); + insert_nvt_preferences_list (preferences, truncate); g_list_free_full (preferences, g_free); nvti_free (nvti); children = next_entities (children); } + if (truncate) { + sql ("DROP VIEW vulns;"); + sql ("DROP TABLE nvts, nvt_preferences, vt_refs, vt_severities;"); + + sql ("ALTER TABLE vt_refs_rebuild RENAME TO vt_refs;"); + sql ("ALTER TABLE vt_severities_rebuild RENAME TO vt_severities;"); + sql ("ALTER TABLE nvt_preferences_rebuild RENAME TO nvt_preferences;"); + sql ("ALTER TABLE nvts_rebuild RENAME TO nvts;"); + + create_view_vulns (); + } + set_nvts_check_time (count_new_vts, count_modified_vts); set_nvts_feed_version (scanner_feed_version); From bdd5168d5715c5f3fea4d5171cb0ada9a9b97818 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 17 Apr 2023 17:31:16 +0200 Subject: [PATCH 2/3] Use better variable name --- src/manage_sql_nvts.c | 58 +++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/manage_sql_nvts.c b/src/manage_sql_nvts.c index 26ce909b2..02409aa61 100644 --- a/src/manage_sql_nvts.c +++ b/src/manage_sql_nvts.c @@ -267,16 +267,16 @@ find_nvt (const char* oid, nvt_t* nvt) * @brief Insert vt_refs for an NVT. * * @param[in] nvti NVT Information. - * @param[in] truncate True if NVT tables were truncated. + * @param[in] rebuild True if rebuilding. */ static void -insert_vt_refs (const nvti_t *nvti, int truncate) +insert_vt_refs (const nvti_t *nvti, int rebuild) { int i; - if (truncate == 0) + if (rebuild == 0) sql ("DELETE FROM vt_refs%s where vt_oid = '%s';", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti)); for (i = 0; i < nvti_vtref_len (nvti); i++) @@ -291,7 +291,7 @@ insert_vt_refs (const nvti_t *nvti, int truncate) sql ("INSERT into vt_refs%s (vt_oid, type, ref_id, ref_text)" " VALUES ('%s', '%s', '%s', '%s');", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti), quoted_type, quoted_id, quoted_text); g_free (quoted_type); @@ -304,19 +304,19 @@ insert_vt_refs (const nvti_t *nvti, int truncate) * @brief Insert vt_severities for an NVT. * * @param[in] nvti NVT Information. - * @param[in] truncate True if NVT tables were truncated. + * @param[in] rebuild True if rebuilding. * * @return Highest severity. */ static double -insert_vt_severities (const nvti_t *nvti, int truncate) +insert_vt_severities (const nvti_t *nvti, int rebuild) { int i; double highest; - if (truncate == 0) + if (rebuild == 0) sql ("DELETE FROM vt_severities%s where vt_oid = '%s';", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti)); highest = 0; @@ -335,7 +335,7 @@ insert_vt_severities (const nvti_t *nvti, int truncate) sql ("INSERT into vt_severities%s (vt_oid, type, origin, date, score," " value)" " VALUES ('%s', '%s', '%s', %i, %0.1f, '%s');", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti), vtseverity_type (severity), quoted_origin, vtseverity_date (severity), vtseverity_score (severity), quoted_value); @@ -355,10 +355,10 @@ insert_vt_severities (const nvti_t *nvti, int truncate) * Always called within a transaction. * * @param[in] nvti NVT Information. - * @param[in] truncate True if NVT tables were truncated. + * @param[in] rebuild True if rebuilding. */ static void -insert_nvt (const nvti_t *nvti, int truncate) +insert_nvt (const nvti_t *nvti, int rebuild) { gchar *qod_str, *qod_type, *cve; gchar *quoted_name, *quoted_summary, *quoted_insight, *quoted_affected; @@ -401,16 +401,16 @@ insert_nvt (const nvti_t *nvti, int truncate) quoted_family = sql_quote (nvti_family (nvti) ? nvti_family (nvti) : ""); - if ((truncate == 0) + if ((rebuild == 0) && sql_int ("SELECT EXISTS (SELECT * FROM nvts WHERE oid = '%s');", nvti_oid (nvti))) sql ("DELETE FROM nvts%s WHERE oid = '%s';", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti)); - insert_vt_refs(nvti, truncate); + insert_vt_refs(nvti, rebuild); - highest = insert_vt_severities(nvti, truncate); + highest = insert_vt_severities(nvti, rebuild); sql ("INSERT into nvts%s (oid, name, summary, insight, affected," " impact, cve, tag, category, family, cvss_base," @@ -418,7 +418,7 @@ insert_nvt (const nvti_t *nvti, int truncate) " solution_method, solution, detection, qod, qod_type)" " VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s'," " '%s', %i, '%s', %0.1f, %i, %i, '%s', '%s', '%s', '%s', '%s', %d, '%s');", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti), quoted_name, quoted_summary, quoted_insight, quoted_affected, quoted_impact, quoted_cve, quoted_tag, nvti_category (nvti), quoted_family, highest, @@ -1529,14 +1529,14 @@ nvti_from_vt (entity_t vt) * * @param[in] get_vts_response OSP GET_VTS response. * @param[in] scanner_feed_version Version of feed from scanner. - * @param[in] truncate Whether to truncate the NVT tables first. + * @param[in] rebuild Whether we're rebuilding the tables. * * @return 0 success, 1 VT integrity check failed, -1 error */ static int update_nvts_from_vts (entity_t *get_vts_response, const gchar *scanner_feed_version, - int truncate) + int rebuild) { entity_t vts, vt; entities_t children; @@ -1561,7 +1561,7 @@ update_nvts_from_vts (entity_t *get_vts_response, sql_begin_immediate (); - if (truncate) { + if (rebuild) { sql ("DROP TABLE IF EXISTS vt_refs_rebuild;"); sql ("DROP TABLE IF EXISTS vt_severities_rebuild;"); sql ("DROP TABLE IF EXISTS nvt_preferences_rebuild;"); @@ -1601,7 +1601,7 @@ update_nvts_from_vts (entity_t *get_vts_response, else count_modified_vts += 1; - insert_nvt (nvti, truncate); + insert_nvt (nvti, rebuild); preferences = NULL; if (update_preferences_from_vt (vt, nvti_oid (nvti), &preferences)) @@ -1609,18 +1609,18 @@ update_nvts_from_vts (entity_t *get_vts_response, sql_rollback (); return -1; } - if (truncate == 0) + if (rebuild == 0) sql ("DELETE FROM nvt_preferences%s WHERE name LIKE '%s:%%';", - truncate ? "_rebuild" : "", + rebuild ? "_rebuild" : "", nvti_oid (nvti)); - insert_nvt_preferences_list (preferences, truncate); + insert_nvt_preferences_list (preferences, rebuild); g_list_free_full (preferences, g_free); nvti_free (nvti); children = next_entities (children); } - if (truncate) { + if (rebuild) { sql ("DROP VIEW vulns;"); sql ("DROP TABLE nvts, nvt_preferences, vt_refs, vt_severities;"); @@ -1859,13 +1859,13 @@ DEF_ACCESS (nvt_severity_iterator_value, 4); * @param[in] update_socket Socket to use to contact scanner. * @param[in] db_feed_version Feed version from meta table. * @param[in] scanner_feed_version Feed version from scanner. - * @param[in] truncate Whether to truncate the NVT tables first. + * @param[in] rebuild Whether to rebuild the NVT tables from scratch. * * @return 0 success, 1 VT integrity check failed, -1 error. */ static int update_nvt_cache_osp (const gchar *update_socket, gchar *db_feed_version, - gchar *scanner_feed_version, int truncate) + gchar *scanner_feed_version, int rebuild) { osp_connection_t *connection; GSList *scanner_prefs; @@ -1874,7 +1874,7 @@ update_nvt_cache_osp (const gchar *update_socket, gchar *db_feed_version, time_t old_nvts_last_modified; int ret; - if (truncate + if (rebuild || db_feed_version == NULL || strcmp (db_feed_version, "") == 0 || strcmp (db_feed_version, "0") == 0) @@ -1907,7 +1907,7 @@ update_nvt_cache_osp (const gchar *update_socket, gchar *db_feed_version, g_free (get_vts_opts.filter); osp_connection_close (connection); - ret = update_nvts_from_vts (&vts, scanner_feed_version, truncate); + ret = update_nvts_from_vts (&vts, scanner_feed_version, rebuild); free_entity (vts); if (ret) return ret; From bd6efb108492bbbbf0d3ded1986ba29269ea6452 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 17 Apr 2023 17:37:12 +0200 Subject: [PATCH 3/3] Remove stray page break --- src/manage_pg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/manage_pg.c b/src/manage_pg.c index 6acb4a903..41b7a4988 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -40,7 +40,7 @@ /* Headers */ - int +int check_db_extensions ();