From d18e6c1c8dd3c31692cf28d0536840a5188ac2a0 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 20 Nov 2023 14:06:59 +0200 Subject: [PATCH 01/46] Change: move GET iterator time conversion out of SQL --- src/manage_get.c | 36 ++++++++++++++++++++++++++++++++---- src/manage_get.h | 4 ++-- src/manage_sql.h | 10 +++++----- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/manage_get.c b/src/manage_get.c index 2b81ac2f8..3c9e151b9 100644 --- a/src/manage_get.c +++ b/src/manage_get.c @@ -154,18 +154,46 @@ get_iterator_comment (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Creation time of the resource or NULL if iteration is complete. + * @return Creation time, or NULL if iteration is complete. Caller must free. */ -DEF_ACCESS (get_iterator_creation_time, 4); +gchar * +get_iterator_creation_time (iterator_t* iterator) +{ + time_t epoch; + char *iso; + + if (iterator->done) return NULL; + + epoch = iterator_int64 (iterator, 4); + iso = iso_time (&epoch); + if (iso) + // iso points to static memory. + return g_strdup (iso); + return g_strdup("ERR"); +} /** * @brief Get the modification time of the resource from a GET iterator. * * @param[in] iterator Iterator. * - * @return Modification time of the resource or NULL if iteration is complete. + * @return Modification time, or NULL if iteration is complete. Caller must free. */ -DEF_ACCESS (get_iterator_modification_time, 5); +gchar * +get_iterator_modification_time (iterator_t* iterator) +{ + time_t epoch; + char *iso; + + if (iterator->done) return NULL; + + epoch = iterator_int64 (iterator, 5); + iso = iso_time (&epoch); + if (iso) + // iso points to static memory. + return g_strdup (iso); + return g_strdup("ERR"); +} /** * @brief Get the owner name of the resource from a GET iterator. diff --git a/src/manage_get.h b/src/manage_get.h index 5c44e667c..a990825f1 100644 --- a/src/manage_get.h +++ b/src/manage_get.h @@ -66,10 +66,10 @@ get_iterator_name (iterator_t*); const char* get_iterator_comment (iterator_t*); -const char* +gchar* get_iterator_creation_time (iterator_t*); -const char* +gchar* get_iterator_modification_time (iterator_t*); const char* diff --git a/src/manage_sql.h b/src/manage_sql.h index 6ea3ac24c..e897075c1 100644 --- a/src/manage_sql.h +++ b/src/manage_sql.h @@ -248,9 +248,9 @@ typedef struct * * @param[in] prefix Column prefix. */ -#define GET_ITERATOR_COLUMNS_STRING \ - "id, uuid, name, comment, iso_time (creation_time)," \ - " iso_time (modification_time), creation_time AS created," \ +#define GET_ITERATOR_COLUMNS_STRING \ + "id, uuid, name, comment, creation_time," \ + " modification_time, creation_time AS created," \ " modification_time AS modified" /** @@ -263,8 +263,8 @@ typedef struct { prefix "uuid", NULL, KEYWORD_TYPE_STRING }, \ { prefix "name", NULL, KEYWORD_TYPE_STRING }, \ { prefix "comment", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (" prefix "creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (" prefix "modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { prefix "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { prefix "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { prefix "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { prefix "modification_time", "modified", KEYWORD_TYPE_INTEGER } From 82b6b6d8e9c83c63ba4c775f549d0c86c110b08a Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 20 Nov 2023 15:46:29 +0200 Subject: [PATCH 02/46] Free returns from time accessors --- src/gmp.c | 105 +++++++++++++++++++++++++++++++++++--------------- src/gmp_get.c | 15 +++++--- src/manage.c | 15 +++++--- 3 files changed, 93 insertions(+), 42 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index 4918287eb..ab77633d6 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -8040,9 +8040,14 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, if (include_notes_details == 0) { - const char *text = note_iterator_text (notes); - gchar *excerpt = utf8_substring (text, 0, - setting_excerpt_size_int ()); + gchar *excerpt, *creation, *modification; + const char *text; + + text = note_iterator_text (notes); + excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); + creation = get_iterator_creation_time (notes); + modification = get_iterator_modification_time (notes); + /* This must match send_get_common. */ buffer_xml_append_printf (buffer, "%s" @@ -8063,8 +8068,8 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, note_iterator_nvt_oid (notes), note_iterator_nvt_name (notes), note_iterator_nvt_type (notes), - get_iterator_creation_time (notes), - get_iterator_modification_time (notes), + creation, + modification, note_iterator_active (notes), strlen (excerpt) < strlen (text), excerpt, @@ -8072,6 +8077,8 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, && (uuid_task == NULL)) || (note_iterator_result (notes) && (uuid_result == NULL)))); + g_free (creation); + g_free (modification); if (tag_count) { @@ -8092,6 +8099,7 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, int trash_task; time_t end_time; iterator_t tags; + gchar *creation, *modification; if (uuid_task) { @@ -8105,6 +8113,8 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, } end_time = note_iterator_end_time (notes); + creation = get_iterator_creation_time (notes); + modification = get_iterator_modification_time (notes); /* This must match send_get_common. */ buffer_xml_append_printf @@ -8132,8 +8142,8 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, note_iterator_nvt_oid (notes), note_iterator_nvt_name (notes), note_iterator_nvt_type (notes), - get_iterator_creation_time (notes), - get_iterator_modification_time (notes), + creation, + modification, note_iterator_active (notes), end_time > 1 ? iso_time (&end_time) : "", note_iterator_text (notes), @@ -8150,6 +8160,8 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, || (note_iterator_result (notes) && (uuid_result == NULL)))); free (name_task); + g_free (creation); + g_free (modification); if (include_result && uuid_result && note_iterator_result (notes)) { @@ -8304,9 +8316,14 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, if (include_overrides_details == 0) { - const char *text = override_iterator_text (overrides); - gchar *excerpt = utf8_substring (text, 0, - setting_excerpt_size_int ()); + gchar *excerpt, *creation, *modification; + const char *text; + + text = override_iterator_text (overrides); + excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); + creation = get_iterator_creation_time (overrides); + modification = get_iterator_modification_time (overrides); + /* This must match send_get_common. */ buffer_xml_append_printf (buffer, "%s" @@ -8331,8 +8348,8 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, override_iterator_nvt_oid (overrides), override_iterator_nvt_name (overrides), override_iterator_nvt_type (overrides), - get_iterator_creation_time (overrides), - get_iterator_modification_time (overrides), + creation, + modification, override_iterator_active (overrides), strlen (excerpt) < strlen (text), excerpt, @@ -8349,6 +8366,9 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, || (override_iterator_result (overrides) && (uuid_result == NULL)))); + g_free (creation); + g_free (modification); + if (tag_count) { buffer_xml_append_printf (buffer, @@ -8368,6 +8388,7 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, int trash_task; time_t end_time; iterator_t tags; + gchar *creation, *modification; if (uuid_task) { @@ -8381,6 +8402,8 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, } end_time = override_iterator_end_time (overrides); + creation = get_iterator_creation_time (overrides); + modification = get_iterator_modification_time (overrides); /* This must match send_get_common. */ buffer_xml_append_printf @@ -8411,8 +8434,8 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, override_iterator_nvt_oid (overrides), override_iterator_nvt_name (overrides), override_iterator_nvt_type (overrides), - get_iterator_creation_time (overrides), - get_iterator_modification_time (overrides), + creation, + modification, override_iterator_active (overrides), end_time > 1 ? iso_time (&end_time) : "", override_iterator_text (overrides), @@ -8433,6 +8456,8 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, || (override_iterator_result (overrides) && (uuid_result == NULL)))); free (name_task); + g_free (creation); + g_free (modification); if (include_result && uuid_result && override_iterator_result (overrides)) @@ -9212,7 +9237,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, int changed, int cert_loaded, int lean, int use_delta_fields) { - const char *descr, *name, *comment, *creation_time; + const char *descr, *name, *comment; const char *severity, *original_severity, *original_level; const char *host, *hostname, *result_id, *port, *path, *asset_id, *qod, *qod_type; char *detect_oid, *detect_ref, *detect_cpe, *detect_loc, *detect_name; @@ -9221,6 +9246,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, result_t result; report_t report; task_t selected_task; + gchar *creation_time; comment = get_iterator_comment (results); name = get_iterator_name (results); @@ -9238,7 +9264,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, qod = result_iterator_delta_qod (results); qod_type = result_iterator_delta_qod_type (results); result = result_iterator_delta_result (results); - creation_time = result_iterator_delta_creation_time (results); + creation_time = (gchar *) result_iterator_delta_creation_time (results); // FIX result_id = result_iterator_delta_uuid (results); path = result_iterator_delta_path (results); report = result_iterator_delta_report (results); @@ -9323,9 +9349,12 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, comment); if (creation_time) - buffer_xml_append_printf (buffer, - "%s", - creation_time); + { + buffer_xml_append_printf (buffer, + "%s", + creation_time); + g_free (creation_time); + } if (include_details) { @@ -11536,7 +11565,7 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) while (next (&identifiers)) { const char *source_type; - gchar *name; + gchar *name, *creation, *modification; source_type = host_identifier_iterator_source_type (&identifiers); @@ -11546,6 +11575,9 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) else name = NULL; + creation = get_iterator_creation_time (&identifiers); + modification = get_iterator_modification_time (&identifiers); + xml_string_append (result, "" "%s" @@ -11561,8 +11593,8 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) get_iterator_uuid (&identifiers), get_iterator_name (&identifiers), host_identifier_iterator_value (&identifiers), - get_iterator_creation_time (&identifiers), - get_iterator_modification_time (&identifiers), + creation, + modification, host_identifier_iterator_source_id (&identifiers), source_type, @@ -11573,6 +11605,8 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) name ? name : ""); g_free (name); + g_free (creation); + g_free (modification); if (strcmp (get_iterator_name (&identifiers), "OS") == 0) xml_string_append (result, @@ -14593,6 +14627,10 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) if (get_reports_data->alert_id == NULL) { task_t task; + gchar *creation, *modification; + + creation = get_iterator_creation_time (&reports); + modification = get_iterator_modification_time (&reports); /* Send the standard elements. Should match send_get_common. */ buffer_xml_append_printf @@ -14615,12 +14653,12 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) get_iterator_comment (&reports) ? get_iterator_comment (&reports) : "", - get_iterator_creation_time (&reports) - ? get_iterator_creation_time (&reports) - : "", - get_iterator_modification_time (&reports) - ? get_iterator_modification_time (&reports) - : ""); + creation ? creation : "", + modification ? creation : ""); + + g_free (creation); + g_free (modification); + /* Send short task and report format info */ report_task (report, &task); if (task) @@ -17892,6 +17930,10 @@ handle_get_vulns (gmp_parser_t *gmp_parser, GError **error) while (next (&vulns)) { time_t oldest, newest; + gchar *creation, *modification; + + creation = get_iterator_creation_time (&vulns); + modification = get_iterator_modification_time (&vulns); count ++; SENDF_TO_CLIENT_OR_FAIL ("" @@ -17904,11 +17946,14 @@ handle_get_vulns (gmp_parser_t *gmp_parser, GError **error) get_iterator_uuid (&vulns), get_iterator_name (&vulns), vuln_iterator_type (&vulns), - get_iterator_creation_time (&vulns), - get_iterator_modification_time (&vulns), + creation, + modification, vuln_iterator_severity (&vulns), vuln_iterator_qod (&vulns)); + g_free (creation); + g_free (modification); + // results for the vulnerability oldest = vuln_iterator_oldest (&vulns); SENDF_TO_CLIENT_OR_FAIL ("" diff --git a/src/gmp_get.c b/src/gmp_get.c index b13e73192..32419aef1 100644 --- a/src/gmp_get.c +++ b/src/gmp_get.c @@ -316,9 +316,13 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, const char *tag_type; iterator_t tags; int tag_count; + gchar *creation, *modification; buffer = g_string_new (""); + creation = get_iterator_creation_time (iterator); + modification = get_iterator_modification_time (iterator); + buffer_xml_append_printf (buffer, "<%s id=\"%s\">" "%s" @@ -342,15 +346,14 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, get_iterator_comment (iterator) ? get_iterator_comment (iterator) : "", - get_iterator_creation_time (iterator) - ? get_iterator_creation_time (iterator) - : "", - get_iterator_modification_time (iterator) - ? get_iterator_modification_time (iterator) - : "", + creation ? creation : "", + modification ? modification : "", writable, in_use); + g_free (creation); + g_free (modification); + if (/* The user is the owner. */ (current_credentials.username && get_iterator_owner_name (iterator) diff --git a/src/manage.c b/src/manage.c index 5c8357459..5e94ec03b 100644 --- a/src/manage.c +++ b/src/manage.c @@ -5771,6 +5771,7 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, { char *default_timeout; GString *nvt_tags; + gchar *creation, *modification; DEF (tag); @@ -5830,6 +5831,9 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, } default_timeout = nvt_default_timeout (oid); + creation = get_iterator_creation_time (nvts); + modification = get_iterator_modification_time (nvts); + g_string_append_printf (buffer, "%s" "%s" @@ -5843,19 +5847,18 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, "%s" "%s", default_timeout ? default_timeout : "", - get_iterator_creation_time (nvts) - ? get_iterator_creation_time (nvts) - : "", - get_iterator_modification_time (nvts) - ? get_iterator_modification_time (nvts) - : "", + creation ? creation : "", + modification ? modification : "", nvt_iterator_category (nvts), family_text, nvt_iterator_qod (nvts), nvt_iterator_qod_type (nvts), refs_str->str, nvt_tags->str); + free (default_timeout); + g_free (creation); + g_free (modification); g_string_free (nvt_tags, 1); } From ab20369077ab141d270eb60b0040329420636d65 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 20 Nov 2023 22:17:32 +0200 Subject: [PATCH 03/46] Remove iso_time from iterators that use the GET accessors --- src/manage_sql.c | 24 ++++++++++++------------ src/manage_sql_report_formats.c | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index b18752a75..c2a7206ff 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -21658,8 +21658,8 @@ report_add_results_array (report_t report, GArray *results) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "iso_time (creation_time)", "name", KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = reports.owner)", \ @@ -50006,8 +50006,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, if (host) init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time," " modification_time, owner, owner, value," " source_type, source_id, source_data," " (CASE WHEN source_type LIKE 'Report%%'" @@ -50019,8 +50019,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, " FROM host_identifiers" " WHERE host = %llu" " UNION" - " SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," + " SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time," " modification_time, owner, owner," " (SELECT name FROM oss WHERE id = os)," " source_type, source_id, source_data," @@ -50040,8 +50040,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, ascending ? "ASC" : "DESC"); else init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time," " modification_time, owner, owner, value," " source_type, source_id, source_data, 0, '', ''" " FROM host_identifiers" @@ -50691,8 +50691,8 @@ init_os_host_iterator (iterator_t* iterator, resource_t os) { assert (os); init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time," " modification_time, owner, owner," " (SELECT round (CAST (severity AS numeric), 1)" " FROM host_max_severities" @@ -55046,8 +55046,8 @@ user_resources_in_use (user_t user, { "uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "name", "name", KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "cast (null AS text)", "_owner", KEYWORD_TYPE_INTEGER }, \ diff --git a/src/manage_sql_report_formats.c b/src/manage_sql_report_formats.c index 89b23c4ca..14f2d4e33 100644 --- a/src/manage_sql_report_formats.c +++ b/src/manage_sql_report_formats.c @@ -2599,8 +2599,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -2630,8 +2630,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ From 8f52e13aeb822197761bf87c78b5d29f311244fc Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Wed, 22 Nov 2023 14:06:48 +0200 Subject: [PATCH 04/46] Move iso_time to C for result_iterator_delta_creation_time --- src/gmp.c | 2 +- src/manage.h | 2 +- src/manage_sql.c | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index ab77633d6..b4f65a221 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -9264,7 +9264,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, qod = result_iterator_delta_qod (results); qod_type = result_iterator_delta_qod_type (results); result = result_iterator_delta_result (results); - creation_time = (gchar *) result_iterator_delta_creation_time (results); // FIX + creation_time = result_iterator_delta_creation_time (results); result_id = result_iterator_delta_uuid (results); path = result_iterator_delta_path (results); report = result_iterator_delta_report (results); diff --git a/src/manage.h b/src/manage.h index a4936ea7c..05b72b536 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1560,7 +1560,7 @@ result_iterator_delta_uuid (iterator_t*); const char * result_iterator_delta_qod_type (iterator_t*); -const char * +gchar * result_iterator_delta_creation_time (iterator_t*); const char * diff --git a/src/manage_sql.c b/src/manage_sql.c index c2a7206ff..d93fe7865 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -22381,7 +22381,7 @@ where_qod (int min_qod) { "comparison.delta_qod", NULL, KEYWORD_TYPE_INTEGER }, \ { "comparison.delta_uuid", NULL, KEYWORD_TYPE_STRING }, \ { "delta_qod_type", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (delta_date, opts.user_zone)", \ + { "delta_date", \ "delta_creation_time", \ KEYWORD_TYPE_STRING }, \ { " iso_time (delta_date, opts.user_zone)", \ @@ -23798,13 +23798,22 @@ result_iterator_delta_qod_type (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return delta creation time if any, else NULL. + * @return Time, or NULL if iteration is complete. Caller must free. */ -const char * +gchar * result_iterator_delta_creation_time (iterator_t* iterator) { - if (iterator->done) return 0; - return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); + time_t epoch; + char *iso; + + if (iterator->done) return NULL; + + epoch = iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); + iso = iso_time (&epoch); + if (iso) + // iso points to static memory. + return g_strdup (iso); + return g_strdup("ERR"); } /** From b10666a2afa54315d67cea535dfbd443a57c246e Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Wed, 22 Nov 2023 14:27:51 +0200 Subject: [PATCH 05/46] Move iso_time to C for result_iterator_delta_modification_time --- src/manage.h | 2 +- src/manage_sql.c | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/manage.h b/src/manage.h index 05b72b536..f0257416a 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1563,7 +1563,7 @@ result_iterator_delta_qod_type (iterator_t*); gchar * result_iterator_delta_creation_time (iterator_t*); -const char * +gchar * result_iterator_delta_modification_time (iterator_t*); task_t diff --git a/src/manage_sql.c b/src/manage_sql.c index d93fe7865..9dfe2c7d9 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -22384,7 +22384,7 @@ where_qod (int min_qod) { "delta_date", \ "delta_creation_time", \ KEYWORD_TYPE_STRING }, \ - { " iso_time (delta_date, opts.user_zone)", \ + { "delta_date", \ "delta_modification_time", \ KEYWORD_TYPE_STRING }, \ { "delta_task", NULL, KEYWORD_TYPE_INTEGER }, \ @@ -23821,13 +23821,22 @@ result_iterator_delta_creation_time (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return delta modification time if any, else NULL. + * @return Time, or NULL if iteration is complete. Caller must free. */ -const char * +gchar * result_iterator_delta_modification_time (iterator_t* iterator) { - if (iterator->done) return 0; - return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); + time_t epoch; + char *iso; + + if (iterator->done) return NULL; + + epoch = iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); + iso = iso_time (&epoch); + if (iso) + // iso points to static memory. + return g_strdup (iso); + return g_strdup("ERR"); } /** From f06311c29e67f069107b41410e67484c9edbb9e2 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Wed, 22 Nov 2023 16:17:07 +0200 Subject: [PATCH 06/46] Fix column keyword types --- src/manage_sql.c | 12 ++++++------ src/manage_sql_report_formats.c | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 9dfe2c7d9..97230d2b5 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -21658,8 +21658,8 @@ report_add_results_array (report_t report, GArray *results) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "iso_time (creation_time)", "name", KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = reports.owner)", \ @@ -22383,10 +22383,10 @@ where_qod (int min_qod) { "delta_qod_type", NULL, KEYWORD_TYPE_STRING }, \ { "delta_date", \ "delta_creation_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "delta_date", \ "delta_modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "delta_task", NULL, KEYWORD_TYPE_INTEGER }, \ { "delta_report", NULL, KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -55064,8 +55064,8 @@ user_resources_in_use (user_t user, { "uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "name", "name", KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "cast (null AS text)", "_owner", KEYWORD_TYPE_INTEGER }, \ diff --git a/src/manage_sql_report_formats.c b/src/manage_sql_report_formats.c index 14f2d4e33..5aff23131 100644 --- a/src/manage_sql_report_formats.c +++ b/src/manage_sql_report_formats.c @@ -2599,8 +2599,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -2630,8 +2630,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ From 1e1e1475470c8816973df3fbf56d4e54fb09031d Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 12 Dec 2023 15:16:19 +0200 Subject: [PATCH 07/46] Return time from get_iterator_creation_time and get_iterator_modification_time --- src/gmp.c | 246 ++++++++++++++++++++++++----------------------- src/gmp_get.c | 29 +++--- src/manage.c | 17 ++-- src/manage.h | 4 +- src/manage_get.c | 34 ++----- src/manage_get.h | 4 +- src/manage_sql.c | 34 ++----- src/utils.c | 20 +++- src/utils.h | 3 + 9 files changed, 189 insertions(+), 202 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index 25fc54183..17f2335e5 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -8083,36 +8083,41 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, if (include_notes_details == 0) { - gchar *excerpt, *creation, *modification; + gchar *excerpt; const char *text; text = note_iterator_text (notes); excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); - creation = get_iterator_creation_time (notes); - modification = get_iterator_modification_time (notes); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" - "1" - "0" - "%i" - "%s" - "%i", + "", get_iterator_owner_name (notes) ? get_iterator_owner_name (notes) : "", note_iterator_nvt_oid (notes), note_iterator_nvt_name (notes), - note_iterator_nvt_type (notes), - creation, - modification, + note_iterator_nvt_type (notes)); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (notes))); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (notes))); + + buffer_xml_append_printf (buffer, + "1" + "0" + "%i" + "%s" + "%i", note_iterator_active (notes), strlen (excerpt) < strlen (text), excerpt, @@ -8120,8 +8125,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, && (uuid_task == NULL)) || (note_iterator_result (notes) && (uuid_result == NULL)))); - g_free (creation); - g_free (modification); if (tag_count) { @@ -8142,7 +8145,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, int trash_task; time_t end_time; iterator_t tags; - gchar *creation, *modification; if (uuid_task) { @@ -8156,19 +8158,35 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, } end_time = note_iterator_end_time (notes); - creation = get_iterator_creation_time (notes); - modification = get_iterator_modification_time (notes); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (notes) + ? get_iterator_owner_name (notes) + : "", + note_iterator_nvt_oid (notes), + note_iterator_nvt_name (notes), + note_iterator_nvt_type (notes)); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_creation_time (notes))); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_modification_time (notes))); + + buffer_xml_append_printf + (buffer, "1" "0" "%i" @@ -8179,14 +8197,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, "%s" "%s%i" "%i", - get_iterator_owner_name (notes) - ? get_iterator_owner_name (notes) - : "", - note_iterator_nvt_oid (notes), - note_iterator_nvt_name (notes), - note_iterator_nvt_type (notes), - creation, - modification, note_iterator_active (notes), end_time > 1 ? iso_time (&end_time) : "", note_iterator_text (notes), @@ -8203,8 +8213,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, || (note_iterator_result (notes) && (uuid_result == NULL)))); free (name_task); - g_free (creation); - g_free (modification); if (include_result && uuid_result && note_iterator_result (notes)) { @@ -8359,23 +8367,36 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, if (include_overrides_details == 0) { - gchar *excerpt, *creation, *modification; + gchar *excerpt; const char *text; text = override_iterator_text (overrides); excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); - creation = get_iterator_creation_time (overrides); - modification = get_iterator_modification_time (overrides); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (overrides) + ? get_iterator_owner_name (overrides) + : "", + override_iterator_nvt_oid (overrides), + override_iterator_nvt_name (overrides), + override_iterator_nvt_type (overrides)); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (overrides))); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (overrides))); + + buffer_xml_append_printf (buffer, "1" "0" "%i" @@ -8385,14 +8406,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, "%s" "%s" "%i", - get_iterator_owner_name (overrides) - ? get_iterator_owner_name (overrides) - : "", - override_iterator_nvt_oid (overrides), - override_iterator_nvt_name (overrides), - override_iterator_nvt_type (overrides), - creation, - modification, override_iterator_active (overrides), strlen (excerpt) < strlen (text), excerpt, @@ -8409,9 +8422,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, || (override_iterator_result (overrides) && (uuid_result == NULL)))); - g_free (creation); - g_free (modification); - if (tag_count) { buffer_xml_append_printf (buffer, @@ -8431,7 +8441,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, int trash_task; time_t end_time; iterator_t tags; - gchar *creation, *modification; if (uuid_task) { @@ -8445,19 +8454,35 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, } end_time = override_iterator_end_time (overrides); - creation = get_iterator_creation_time (overrides); - modification = get_iterator_modification_time (overrides); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (overrides) + ? get_iterator_owner_name (overrides) + : "", + override_iterator_nvt_oid (overrides), + override_iterator_nvt_name (overrides), + override_iterator_nvt_type (overrides)); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_creation_time (overrides))); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_modification_time (overrides))); + + buffer_xml_append_printf + (buffer, "1" "0" "%i" @@ -8471,14 +8496,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, "%s" "%s%i" "%i", - get_iterator_owner_name (overrides) - ? get_iterator_owner_name (overrides) - : "", - override_iterator_nvt_oid (overrides), - override_iterator_nvt_name (overrides), - override_iterator_nvt_type (overrides), - creation, - modification, override_iterator_active (overrides), end_time > 1 ? iso_time (&end_time) : "", override_iterator_text (overrides), @@ -8499,8 +8516,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, || (override_iterator_result (overrides) && (uuid_result == NULL)))); free (name_task); - g_free (creation); - g_free (modification); if (include_result && uuid_result && override_iterator_result (overrides)) @@ -9289,7 +9304,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, result_t result; report_t report; task_t selected_task; - gchar *creation_time; + time_t creation_time; comment = get_iterator_comment (results); name = get_iterator_name (results); @@ -9361,7 +9376,8 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, if (lean == 0) { - const char *owner_name, *modification_time; + const char *owner_name; + time_t modification_time; if (use_delta_fields) { @@ -9382,7 +9398,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, if (modification_time) buffer_xml_append_printf (buffer, "%s", - modification_time); + iso_time (&modification_time) ?: ""); } if (comment @@ -9392,12 +9408,9 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, comment); if (creation_time) - { - buffer_xml_append_printf (buffer, - "%s", - creation_time); - g_free (creation_time); - } + buffer_xml_append_printf (buffer, + "%s", + iso_time (&creation_time) ?: ""); if (include_details) { @@ -11608,7 +11621,7 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) while (next (&identifiers)) { const char *source_type; - gchar *name, *creation, *modification; + gchar *name; source_type = host_identifier_iterator_source_type (&identifiers); @@ -11618,26 +11631,29 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) else name = NULL; - creation = get_iterator_creation_time (&identifiers); - modification = get_iterator_modification_time (&identifiers); - xml_string_append (result, "" "%s" - "%s" - "%s" - "%s" + "%s", + get_iterator_uuid (&identifiers), + get_iterator_name (&identifiers), + host_identifier_iterator_value (&identifiers)); + + xml_string_append (result, + "%s", + iso_if_time (get_iterator_creation_time (&identifiers))); + + xml_string_append (result, + "%s", + iso_if_time (get_iterator_modification_time (&identifiers))); + + xml_string_append (result, "" "%s" "%s" "%i" "%s" "", - get_iterator_uuid (&identifiers), - get_iterator_name (&identifiers), - host_identifier_iterator_value (&identifiers), - creation, - modification, host_identifier_iterator_source_id (&identifiers), source_type, @@ -11648,8 +11664,6 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) name ? name : ""); g_free (name); - g_free (creation); - g_free (modification); if (strcmp (get_iterator_name (&identifiers), "OS") == 0) xml_string_append (result, @@ -14670,23 +14684,14 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) if (get_reports_data->alert_id == NULL) { task_t task; - gchar *creation, *modification; - - creation = get_iterator_creation_time (&reports); - modification = get_iterator_modification_time (&reports); /* Send the standard elements. Should match send_get_common. */ + buffer_xml_append_printf (prefix, "%s" "%s" - "%s" - "%s" - "" - "%s" - "" - "0" - "0", + "%s", get_iterator_owner_name (&reports) ? get_iterator_owner_name (&reports) : "", @@ -14695,12 +14700,19 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) : "", get_iterator_comment (&reports) ? get_iterator_comment (&reports) - : "", - creation ? creation : "", - modification ? creation : ""); + : ""); - g_free (creation); - g_free (modification); + buffer_xml_append_printf + (prefix, + "%s", + iso_if_time (get_iterator_creation_time (&reports))); + + buffer_xml_append_printf + (prefix, + "%s" + "0" + "0", + iso_if_time (get_iterator_modification_time (&reports))); /* Send short task and report format info */ report_task (report, &task); @@ -18276,30 +18288,26 @@ handle_get_vulns (gmp_parser_t *gmp_parser, GError **error) while (next (&vulns)) { time_t oldest, newest; - gchar *creation, *modification; - - creation = get_iterator_creation_time (&vulns); - modification = get_iterator_modification_time (&vulns); count ++; SENDF_TO_CLIENT_OR_FAIL ("" "%s" - "%s" - "%s" - "%s" - "%1.1f" - "%d", + "%s", get_iterator_uuid (&vulns), get_iterator_name (&vulns), - vuln_iterator_type (&vulns), - creation, - modification, + vuln_iterator_type (&vulns)); + + SENDF_TO_CLIENT_OR_FAIL ("%s", + iso_if_time (get_iterator_creation_time (&vulns))); + + SENDF_TO_CLIENT_OR_FAIL ("%s", + iso_if_time (get_iterator_modification_time (&vulns))); + + SENDF_TO_CLIENT_OR_FAIL ("%1.1f" + "%d", vuln_iterator_severity (&vulns), vuln_iterator_qod (&vulns)); - g_free (creation); - g_free (modification); - // results for the vulnerability oldest = vuln_iterator_oldest (&vulns); SENDF_TO_CLIENT_OR_FAIL ("" diff --git a/src/gmp_get.c b/src/gmp_get.c index 32419aef1..3793a49d6 100644 --- a/src/gmp_get.c +++ b/src/gmp_get.c @@ -316,23 +316,14 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, const char *tag_type; iterator_t tags; int tag_count; - gchar *creation, *modification; buffer = g_string_new (""); - creation = get_iterator_creation_time (iterator); - modification = get_iterator_modification_time (iterator); - buffer_xml_append_printf (buffer, "<%s id=\"%s\">" "%s" "%s" - "%s" - "%s" - "%s" - "%i" - "%i" - "", + "%s", type, get_iterator_uuid (iterator) ? get_iterator_uuid (iterator) @@ -345,15 +336,21 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, : "", get_iterator_comment (iterator) ? get_iterator_comment (iterator) - : "", - creation ? creation : "", - modification ? modification : "", + : ""); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (iterator))); + + buffer_xml_append_printf (buffer, + "%s" + "%i" + "%i" + "", + iso_if_time (get_iterator_modification_time (iterator)), writable, in_use); - g_free (creation); - g_free (modification); - if (/* The user is the owner. */ (current_credentials.username && get_iterator_owner_name (iterator) diff --git a/src/manage.c b/src/manage.c index 5e94ec03b..7d51e0b50 100644 --- a/src/manage.c +++ b/src/manage.c @@ -5771,7 +5771,6 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, { char *default_timeout; GString *nvt_tags; - gchar *creation, *modification; DEF (tag); @@ -5830,14 +5829,18 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, nvt_iterator_detection (nvts)); } + g_string_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (nvts))); + + g_string_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (nvts))); + default_timeout = nvt_default_timeout (oid); - creation = get_iterator_creation_time (nvts); - modification = get_iterator_modification_time (nvts); g_string_append_printf (buffer, "%s" - "%s" - "%s" "%d" "%s" "" @@ -5847,8 +5850,6 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, "%s" "%s", default_timeout ? default_timeout : "", - creation ? creation : "", - modification ? modification : "", nvt_iterator_category (nvts), family_text, nvt_iterator_qod (nvts), @@ -5857,8 +5858,6 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, nvt_tags->str); free (default_timeout); - g_free (creation); - g_free (modification); g_string_free (nvt_tags, 1); } diff --git a/src/manage.h b/src/manage.h index c78954f0a..77977728e 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1562,10 +1562,10 @@ result_iterator_delta_uuid (iterator_t*); const char * result_iterator_delta_qod_type (iterator_t*); -gchar * +time_t result_iterator_delta_creation_time (iterator_t*); -gchar * +time_t result_iterator_delta_modification_time (iterator_t*); task_t diff --git a/src/manage_get.c b/src/manage_get.c index 3c9e151b9..26b32a12c 100644 --- a/src/manage_get.c +++ b/src/manage_get.c @@ -154,22 +154,13 @@ get_iterator_comment (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Creation time, or NULL if iteration is complete. Caller must free. + * @return Creation time, or 0 if iteration is complete. */ -gchar * +time_t get_iterator_creation_time (iterator_t* iterator) { - time_t epoch; - char *iso; - - if (iterator->done) return NULL; - - epoch = iterator_int64 (iterator, 4); - iso = iso_time (&epoch); - if (iso) - // iso points to static memory. - return g_strdup (iso); - return g_strdup("ERR"); + if (iterator->done) return 0; + return iterator_int64 (iterator, 4); } /** @@ -177,22 +168,13 @@ get_iterator_creation_time (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Modification time, or NULL if iteration is complete. Caller must free. + * @return Modification time, or 0 if iteration is complete. */ -gchar * +time_t get_iterator_modification_time (iterator_t* iterator) { - time_t epoch; - char *iso; - - if (iterator->done) return NULL; - - epoch = iterator_int64 (iterator, 5); - iso = iso_time (&epoch); - if (iso) - // iso points to static memory. - return g_strdup (iso); - return g_strdup("ERR"); + if (iterator->done) return 0; + return iterator_int64 (iterator, 5); } /** diff --git a/src/manage_get.h b/src/manage_get.h index a990825f1..2b7bfce24 100644 --- a/src/manage_get.h +++ b/src/manage_get.h @@ -66,10 +66,10 @@ get_iterator_name (iterator_t*); const char* get_iterator_comment (iterator_t*); -gchar* +time_t get_iterator_creation_time (iterator_t*); -gchar* +time_t get_iterator_modification_time (iterator_t*); const char* diff --git a/src/manage_sql.c b/src/manage_sql.c index 4a0f2f5ed..a0882dc01 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -23885,22 +23885,13 @@ result_iterator_delta_qod_type (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Time, or NULL if iteration is complete. Caller must free. + * @return Time, or 0 if iteration is complete. */ -gchar * +time_t result_iterator_delta_creation_time (iterator_t* iterator) { - time_t epoch; - char *iso; - - if (iterator->done) return NULL; - - epoch = iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); - iso = iso_time (&epoch); - if (iso) - // iso points to static memory. - return g_strdup (iso); - return g_strdup("ERR"); + if (iterator->done) return 0; + return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); } /** @@ -23908,22 +23899,13 @@ result_iterator_delta_creation_time (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Time, or NULL if iteration is complete. Caller must free. + * @return Time, or 0 if iteration is complete. */ -gchar * +time_t result_iterator_delta_modification_time (iterator_t* iterator) { - time_t epoch; - char *iso; - - if (iterator->done) return NULL; - - epoch = iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); - iso = iso_time (&epoch); - if (iso) - // iso points to static memory. - return g_strdup (iso); - return g_strdup("ERR"); + if (iterator->done) return 0; + return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); } /** diff --git a/src/utils.c b/src/utils.c index 83c55f7fd..1763b15e6 100644 --- a/src/utils.c +++ b/src/utils.c @@ -483,7 +483,7 @@ iso_time_internal (time_t *epoch_time, const char **abbrev) /** * @brief Create an ISO time from seconds since epoch. * - * @param[in] epoch_time Time in seconds from epoch. + * @param[in] epoch_time Pointer to time in seconds from epoch. * * @return Pointer to ISO time in static memory, or NULL on error. */ @@ -496,7 +496,7 @@ iso_time (time_t *epoch_time) /** * @brief Create an ISO time from seconds since epoch, given a timezone. * - * @param[in] epoch_time Time in seconds from epoch. + * @param[in] epoch_time Pointer to time in seconds from epoch. * @param[in] zone Timezone. * @param[out] abbrev Timezone abbreviation. * @@ -543,6 +543,22 @@ iso_time_tz (time_t *epoch_time, const char *zone, const char **abbrev) return ret; } +/** + * @brief Create an ISO time from seconds since epoch, with a 0 check. + * + * @param[in] epoch_time Time in seconds from epoch. + * + * @return ISO time string in static memory. If epoch_time is 0 then string is empty. + */ +char * +iso_if_time (time_t epoch_time) +{ + static char *empty = ""; + if (epoch_time) + return iso_time (&epoch_time); + return empty; +} + /* Locks. */ diff --git a/src/utils.h b/src/utils.h index 50333d71e..fe8d05433 100644 --- a/src/utils.h +++ b/src/utils.h @@ -55,6 +55,9 @@ iso_time (time_t *); char * iso_time_tz (time_t *, const char *, const char **); +char * +iso_if_time (time_t); + /** * @brief Lockfile. */ From 05ae3c90a0e5906d751289830ce214b4ecc18fb8 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 18 Dec 2023 11:55:36 +0200 Subject: [PATCH 08/46] Remove creation and modification iso_time from result iterator --- src/manage_sql.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index b203ffc41..3e5bc84e1 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -22143,10 +22143,10 @@ where_qod (int min_qod) "name", \ KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "creation_time", \ KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "modification_time", \ KEYWORD_TYPE_STRING }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ @@ -22296,10 +22296,10 @@ where_qod (int min_qod) "name", \ KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "creation_time", \ KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "modification_time", \ KEYWORD_TYPE_STRING }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ From 03a951da95202c07b6ffc663f999d0e1a6bc4f7c Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 18 Dec 2023 12:39:51 +0200 Subject: [PATCH 09/46] Remove creation and modification iso_time from note and override iterators --- src/manage_sql.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 3e5bc84e1..f0d0383f1 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -38521,8 +38521,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "notes.creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "notes.modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "notes.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes.owner)", \ @@ -38576,8 +38576,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, { "notes_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes_trash.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes_trash.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "notes_trash.creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "notes_trash.modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "notes_trash.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes_trash.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes_trash.owner)", \ @@ -39758,8 +39758,8 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "overrides.creation_time", NULL, KEYWORD_TYPE_STRING }, \ + { "overrides.modification_time", NULL, KEYWORD_TYPE_STRING }, \ { "overrides.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "overrides.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -39826,10 +39826,10 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, { "overrides_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides_trash.creation_time)", \ + { "overrides_trash.creation_time", \ NULL, \ KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides_trash.modification_time)", \ + { "overrides_trash.modification_time", \ NULL, \ KEYWORD_TYPE_STRING }, \ { "overrides_trash.creation_time", \ From 3472888f177e1fd4a17b576470699f7d2735ea68 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 18 Dec 2023 12:53:53 +0200 Subject: [PATCH 10/46] Use integer for creation and modification columns in iterators --- src/manage_sql.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index f0d0383f1..0d2412918 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -22145,10 +22145,10 @@ where_qod (int min_qod) { "''", "comment", KEYWORD_TYPE_STRING }, \ { "date", \ "creation_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", \ "modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ { "date", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -22298,10 +22298,10 @@ where_qod (int min_qod) { "''", "comment", KEYWORD_TYPE_STRING }, \ { "date", \ "creation_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", \ "modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ { "date", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -38521,8 +38521,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "notes.creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "notes.modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "notes.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "notes.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "notes.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes.owner)", \ @@ -38576,8 +38576,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, { "notes_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "notes_trash.creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "notes_trash.modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "notes_trash.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "notes_trash.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "notes_trash.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes_trash.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes_trash.owner)", \ @@ -39758,8 +39758,8 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "overrides.creation_time", NULL, KEYWORD_TYPE_STRING }, \ - { "overrides.modification_time", NULL, KEYWORD_TYPE_STRING }, \ + { "overrides.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "overrides.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "overrides.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "overrides.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -39828,10 +39828,10 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "overrides_trash.creation_time", \ NULL, \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "overrides_trash.modification_time", \ NULL, \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "overrides_trash.creation_time", \ "created", \ KEYWORD_TYPE_INTEGER }, \ From f62db8927326c6c628dbae477385f69e03039b4b Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Mon, 18 Dec 2023 13:51:23 +0200 Subject: [PATCH 11/46] Keep columns unique --- src/manage_sql.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 0d2412918..0b3ce888c 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -50136,8 +50136,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, if (host) init_iterator (iterator, "SELECT id, uuid, name, comment, creation_time," - " modification_time, creation_time," - " modification_time, owner, owner, value," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner, value," " source_type, source_id, source_data," " (CASE WHEN source_type LIKE 'Report%%'" " THEN NOT EXISTS (SELECT * FROM reports" @@ -50149,8 +50149,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, " WHERE host = %llu" " UNION" " SELECT id, uuid, name, comment, creation_time," - " modification_time, creation_time," - " modification_time, owner, owner," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner," " (SELECT name FROM oss WHERE id = os)," " source_type, source_id, source_data," " (CASE WHEN source_type LIKE 'Report%%'" @@ -50170,8 +50170,8 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, else init_iterator (iterator, "SELECT id, uuid, name, comment, creation_time," - " modification_time, creation_time," - " modification_time, owner, owner, value," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner, value," " source_type, source_id, source_data, 0, '', ''" " FROM host_identifiers" " ORDER BY %s %s;", From cb983dfada959f0f73bfed7a438e5c79898408d6 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 2 Jan 2024 15:18:12 +0200 Subject: [PATCH 12/46] Work around clang-format limitation --- src/utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.h b/src/utils.h index fe8d05433..dfeed44b6 100644 --- a/src/utils.h +++ b/src/utils.h @@ -56,7 +56,7 @@ char * iso_time_tz (time_t *, const char *, const char **); char * -iso_if_time (time_t); +iso_if_time (time_t epoch_time); /** * @brief Lockfile. From c218648ed5e66c2036e5ff2bd48db898facbc948 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Thu, 22 Feb 2024 10:31:07 +0100 Subject: [PATCH 13/46] Fix: Free quoted strings in SCAP update Some of the quoted strings that were not freed in the CPE and CVE updates now are. --- src/manage_sql_secinfo.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index adc8d3dda..4911b73d8 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2101,6 +2101,9 @@ insert_scap_cpe_details (inserts_t *inserts, element_t cpe_item) quoted_name, quoted_details_xml); + g_free (quoted_name); + g_free (quoted_details_xml); + inserts->current_chunk_size++; return 0; @@ -2670,6 +2673,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, increment_transaction_size (transaction_size); g_free (quoted_summary); g_free (quoted_cvss_vector); + g_free (quoted_software); insert_cve_products (list, cve, time_published, time_modified, hashed_cpes, transaction_size); From f1a7fc86e39c6721ce29fb08081a375fcd644629 Mon Sep 17 00:00:00 2001 From: Greenbone Bot Date: Mon, 26 Feb 2024 11:22:41 +0000 Subject: [PATCH 14/46] Automatic release to 23.4.0 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e09ac5d41..61e4be405 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0) message ("-- Configuring Greenbone Vulnerability Manager...") project (gvm - VERSION 23.3.1 + VERSION 23.4.0 LANGUAGES C) if (POLICY CMP0005) @@ -58,7 +58,7 @@ endif (NOT CMAKE_BUILD_TYPE MATCHES "Release") # Set dev version if this is a development version and not a full release, # unset (put value 0 or delete line) before a full release and reset after. -set (PROJECT_DEV_VERSION 1) +set (PROJECT_DEV_VERSION 0) # If PROJECT_DEV_VERSION is set, the version string will be set to: # "major.minor.patch~dev${PROJECT_DEV_VERSION}${GIT_REVISION}" From 483d0106ee27763450587df668a555d25449e22e Mon Sep 17 00:00:00 2001 From: Greenbone Bot Date: Mon, 26 Feb 2024 11:22:43 +0000 Subject: [PATCH 15/46] Automatic adjustments after release * Update to version 23.4.1-dev1 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61e4be405..a915ed55a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0) message ("-- Configuring Greenbone Vulnerability Manager...") project (gvm - VERSION 23.4.0 + VERSION 23.4.1 LANGUAGES C) if (POLICY CMP0005) @@ -58,7 +58,7 @@ endif (NOT CMAKE_BUILD_TYPE MATCHES "Release") # Set dev version if this is a development version and not a full release, # unset (put value 0 or delete line) before a full release and reset after. -set (PROJECT_DEV_VERSION 0) +set (PROJECT_DEV_VERSION 1) # If PROJECT_DEV_VERSION is set, the version string will be set to: # "major.minor.patch~dev${PROJECT_DEV_VERSION}${GIT_REVISION}" From da49baf1f3650b29f077adaa301fad6ebc4939ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 10:31:30 +0100 Subject: [PATCH 16/46] CI: Rework gvmd build container image Change the gvmd-build container image build to be independent of changes and releases of gvmd itself. The images only need to be rebuild if gvm-libs has changed or if the base container image has new packages for the dependencies. Therefore it isn't required to rebuild them on every release, pull request or push. Now only two flavors of the gvmd-build image exist. First gvmd-build:stable which is based on the latest gvm-libs release and second gvmd-build:edge which is based on the latest changes in gvm-libs. --- .github/workflows/build-container.yml | 41 ++++++--------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index d326f8c14..93b7704d6 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -1,19 +1,6 @@ name: Build Container Image Builds on: - push: - branches: - - main - tags: ["v*"] - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile - pull_request: - branches: - - main - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile workflow_dispatch: repository_dispatch: schedule: @@ -22,13 +9,16 @@ on: jobs: build-images: + strategy: + matrix: + build: + - stable + - edge name: "Build Images" runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: greenbone/actions/is-latest-tag@v3 - id: latest - name: Setup container meta information id: meta uses: docker/metadata-action@v5 @@ -39,30 +29,18 @@ jobs: org.opencontainers.image.base.name=greenbone/gvm-libs flavor: latest=false # no latest container tag for git tags tags: | - # use version, major.minor and major for tags - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - # use edge for default branch - type=edge - - # set label for non-published pull request builds - type=ref,event=pr - - # when a new git tag is created set stable and a latest tags - type=raw,value=latest,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }} - type=raw,value=stable,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }} + type=raw,value=latest,enable=${{ matrix.build == 'stable' }} + type=raw,value=stable,enable=${{ matrix.build == 'stable' }} + type=raw,value=edge,enable=${{ matrix.build == 'edge' }} - name: Set container build options id: container-opts run: | - if [[ "${{ github.ref_type }}" = 'tag' ]]; then + if [[ "${{ matrix.build }}" = 'stable' ]]; then echo "gvm-libs-version=oldstable" >> $GITHUB_OUTPUT else echo "gvm-libs-version=oldstable-edge" >> $GITHUB_OUTPUT fi - name: Login to DockerHub - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -76,7 +54,6 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: ${{ github.event_name != 'pull_request' }} build-args: | GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }} file: .docker/build.Dockerfile From ae90bd30d29b762ce1810eeb2a753dd7c89d3327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 10:43:22 +0100 Subject: [PATCH 17/46] CI: Don't trigger libthea on gvmd-build image changes anymore Libthea project has been abandoned and will not get any further changes. Therefore there is no need to trigger any workflow of libthea. --- .github/workflows/build-container.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 93b7704d6..7a6175c58 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -60,10 +60,3 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Trigger libtheia container build - if: github.event_name != 'pull_request' - run: | - curl -X POST https://api.github.com/repos/greenbone/libtheia/actions/workflows/container.yml/dispatches \ - -H "Accept: application/vnd.github.v3+json" \ - -u greenbonebot:${{ secrets.GREENBONE_BOT_TOKEN }} \ - -d '{"ref":"main"}' From 2893755b8dac6753b1a5acba75ccbb69abab4077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 11:17:02 +0100 Subject: [PATCH 18/46] CI: Update codeowners Matt is a member of gvmd-maintainers now and therefore shouldn't be listed separately. --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2277decec..85dac033f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,6 @@ # default reviewers -* @greenbone/gvmd-maintainers @mattmundell +* @greenbone/gvmd-maintainers # dev ops -.github/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell -.docker/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell +.github/ @greenbone/devops @greenbone/gvmd-maintainers +.docker/ @greenbone/devops @greenbone/gvmd-maintainers From 2801cf23cded475b4de4510e9a5939c401a8c048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 10:57:58 +0100 Subject: [PATCH 19/46] Remove duplicated shebang line in container startup script The line got duplicated likely when the license header was added. --- .docker/start-gvmd.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh index c4ced08d3..fd47f6ded 100644 --- a/.docker/start-gvmd.sh +++ b/.docker/start-gvmd.sh @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/bin/sh - [ -z "$USER" ] && USER="admin" [ -z "$PASSWORD" ] && PASSWORD="admin" [ -z "$GVMD_ARGS" ] && GVMD_ARGS="--listen-mode=666" From 515d312f2528a3bbdf12a648d82717b3313c5c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 10:59:07 +0100 Subject: [PATCH 20/46] Fix indentation in gvmd container startup script Always use spaces instead of tabs. --- .docker/start-gvmd.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh index fd47f6ded..76c0f8ea6 100644 --- a/.docker/start-gvmd.sh +++ b/.docker/start-gvmd.sh @@ -23,19 +23,19 @@ [ -z "$PGRES_DATA"] && PGRES_DATA="/var/lib/postgresql" if [ -n "$GVM_CERTS" ] && [ "$GVM_CERTS" = true ]; then - echo "Generating certs" - gvm-manage-certs -a + echo "Generating certs" + gvm-manage-certs -a fi # check for psql connection FILE=$PGRES_DATA/started until test -f "$FILE"; do - echo "waiting 1 second for ready postgres container" + echo "waiting 1 second for ready postgres container" sleep 1 done until psql -U "$GVMD_USER" -d gvmd -c "SELECT 'connected' as connection"; do - echo "waiting 1 second to retry psql connection" - sleep 1 + echo "waiting 1 second to retry psql connection" + sleep 1 done # migrate db if necessary @@ -49,6 +49,6 @@ gvmd --modify-setting 78eceaec-3385-11ea-b237-28d24461215b --value "$uid" echo "starting gvmd" gvmd $GVMD_ARGS || - (cat /var/log/gvm/gvmd.log && exit 1) + (cat /var/log/gvm/gvmd.log && exit 1) tail -f /var/log/gvm/gvmd.log From 7f3bdb33ebd6109a4d6de5cbfd13246d77af9375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 11:00:59 +0100 Subject: [PATCH 21/46] Fix checking if PGRES_DATA env variable is set in container startup script Testing if the PGRES_DATA env variable is already set missed a space before the closing bracket. Therefore it wasn't possible to override the to be used postgres data directory. When shellcheck warns you, you should give it attention. --- .docker/start-gvmd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh index 76c0f8ea6..6ef724f44 100644 --- a/.docker/start-gvmd.sh +++ b/.docker/start-gvmd.sh @@ -20,7 +20,7 @@ [ -z "$PASSWORD" ] && PASSWORD="admin" [ -z "$GVMD_ARGS" ] && GVMD_ARGS="--listen-mode=666" [ -z "$GVMD_USER" ] && GVMD_USER="gvmd" -[ -z "$PGRES_DATA"] && PGRES_DATA="/var/lib/postgresql" +[ -z "$PGRES_DATA" ] && PGRES_DATA="/var/lib/postgresql" if [ -n "$GVM_CERTS" ] && [ "$GVM_CERTS" = true ]; then echo "Generating certs" From 07b89ac8c68b4f75b0c0a65dc7fe1c8140300b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 27 Feb 2024 11:03:47 +0100 Subject: [PATCH 22/46] Fix getting the uuid of the admin user in container startup script There could be more then one user starting with admin in their names like foo_admin or admin_foo. Therefore fix the grep command to only show the user admin. --- .docker/start-gvmd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh index 6ef724f44..7083b8f81 100644 --- a/.docker/start-gvmd.sh +++ b/.docker/start-gvmd.sh @@ -44,7 +44,7 @@ gvmd --migrate || true gvmd --create-user=$USER --password=$PASSWORD || true # set the feed import owner -uid=$(gvmd --get-users --verbose | grep $USER | awk '{print $2}') +uid=$(gvmd --get-users --verbose | grep "^$USER " | awk '{print $2}') gvmd --modify-setting 78eceaec-3385-11ea-b237-28d24461215b --value "$uid" echo "starting gvmd" From 469a03dc1b6b0eb012855099bd620d3689a86809 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 27 Feb 2024 14:36:44 +0200 Subject: [PATCH 23/46] In iso_if_time also return empty if iso_time returns NULL --- src/utils.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils.c b/src/utils.c index 1763b15e6..efe108b23 100644 --- a/src/utils.c +++ b/src/utils.c @@ -555,7 +555,13 @@ iso_if_time (time_t epoch_time) { static char *empty = ""; if (epoch_time) - return iso_time (&epoch_time); + { + char *ret; + + ret = iso_time (&epoch_time); + if (ret) + return ret; + } return empty; } From be66e2eb308e6d5c15b6ff944ca6da1881766c23 Mon Sep 17 00:00:00 2001 From: Matt Mundell Date: Tue, 27 Feb 2024 16:52:28 +0200 Subject: [PATCH 24/46] Move iso_time to C for report_config iterator --- src/manage_sql_report_configs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/manage_sql_report_configs.c b/src/manage_sql_report_configs.c index ed158db2f..6d4b44c15 100644 --- a/src/manage_sql_report_configs.c +++ b/src/manage_sql_report_configs.c @@ -661,8 +661,8 @@ restore_report_config (const char *report_config_id) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "comment", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -696,8 +696,8 @@ restore_report_config (const char *report_config_id) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "comment", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ From fdace0cfc1b907024923d7186290bd11bd143ee6 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 4 Mar 2024 14:54:15 +0100 Subject: [PATCH 25/46] Fix: Replaced python call - avoid problems with installations. The python call and the function urlencode was replaced by directly assining the manually encoded simple constant strings to the variables to avoid problems with different python installations. --- src/alert_methods/TippingPoint/alert | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/alert_methods/TippingPoint/alert b/src/alert_methods/TippingPoint/alert index fcde932d2..518cb9a32 100644 --- a/src/alert_methods/TippingPoint/alert +++ b/src/alert_methods/TippingPoint/alert @@ -25,12 +25,6 @@ CONVERT_SCRIPT=$4 AUTH_PATH=$5 REPORT_PATH=$6 -# Function to encode for URL -urlencode () { - RET=$(python -c "import urllib, sys; print urllib.quote(sys.argv[1])" "$1") - echo "$RET" -} - # Create temp file for converted report REPORT_DATE=$(xmlstarlet sel -t -v "report/timestamp" < $REPORT_PATH) EXIT_CODE=$? @@ -76,9 +70,9 @@ END_TIME=$(TZ=UTC date -d "$END_TIME" +%Y-%m-%dT%H:%M:%S.000Z) RUNTIME="$START_TIME/$END_TIME" # Upload the report -VENDOR=$(urlencode "Greenbone") -PRODUCT=$(urlencode "Greenbone Vulnerability Manager") -FORMAT_VERSION=$(urlencode "1.0.0") +VENDOR="Greenbone" +PRODUCT="Greenbone%20Vulnerability%20Manager" +FORMAT_VERSION="1.0.0" CN_REPLACEMENT="Tippingpoint" if [ "1" = $CERT_WORKAROUND ] From edf71072fec7b1023ac333329dcb129cc3af2510 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 4 Mar 2024 17:16:10 +0100 Subject: [PATCH 26/46] Small fix of an other bug. --- src/alert_methods/TippingPoint/report-convert.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/alert_methods/TippingPoint/report-convert.py b/src/alert_methods/TippingPoint/report-convert.py index cf4a49322..c2768ff9c 100755 --- a/src/alert_methods/TippingPoint/report-convert.py +++ b/src/alert_methods/TippingPoint/report-convert.py @@ -137,6 +137,10 @@ def convert (xml_tree, out_file): nvt_cve = ''; nvt_elem = result_elem.find ('nvt') nvt_refs = nvt_elem.find ('refs'); + + if (nvt_refs is None): + continue + for ref in nvt_refs.findall('ref'): if (ref.attrib['type'] == 'cve'): if (nvt_cve == ''): From edea0be12345ca6e85571a5494af4233febf45e8 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 5 Mar 2024 15:18:36 +0100 Subject: [PATCH 27/46] Fix: Handle value of last_update and SCAP schema when feed update fails. --- src/manage_sql_secinfo.c | 108 ++++++++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 4911b73d8..4e69d5ab9 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3427,38 +3427,15 @@ update_scap_placeholders () " WHERE cpe=cpes.id))" " WHERE cpes.title IS NULL;"); } - + /** - * @brief Finish scap update. - * - * @return 0 success, -1 error. + * @brief Update CERT data that depends on SCAP. */ -static int -update_scap_end () +static void +update_cert_data () { int cert_db_version; - g_debug ("%s: update timestamp", __func__); - - update_scap_timestamp (); - - /* Replace the real scap schema with the new one. */ - - if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" - " information_schema.schemata" - " WHERE schema_name = 'scap');")) - { - sql ("ALTER SCHEMA scap RENAME TO scap3;"); - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - sql ("DROP SCHEMA scap3 CASCADE;"); - /* View 'vulns' contains references into the SCAP schema, so it is - * removed by the CASCADE. */ - create_view_vulns (); - } - else - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - - /* Update CERT data that depends on SCAP. */ cert_db_version = manage_cert_db_version(); if (cert_db_version == -1) @@ -3490,6 +3467,39 @@ update_scap_end () update_cvss_dfn_cert (1, last_cert_update, last_scap_update); update_cvss_cert_bund (1, last_cert_update, last_scap_update); } +} + +/** + * @brief Finish scap update. + * + * @return 0 success, -1 error. + */ +static int +update_scap_end () +{ + g_debug ("%s: update timestamp", __func__); + + update_scap_timestamp (); + + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + + /* Update CERT data that depends on SCAP. */ + + update_cert_data (); /* Analyze. */ @@ -3503,6 +3513,46 @@ update_scap_end () return 0; } +/** + * @brief Abort scap update. + */ +static void +abort_scap_update () +{ + g_debug ("%s: update timestamp", __func__); + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + update_scap_timestamp (); + sql ("UPDATE scap.meta SET value = " + " (SELECT value from scap2.meta WHERE name = 'last_update')" + " WHERE name = 'last_update';"); + sql ("DROP SCHEMA scap2 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + } + else + { + /* reset scap2 schema */ + manage_db_init ("scap"); + manage_db_init_indexes ("scap"); + manage_db_add_constraints ("scap"); + + update_scap_timestamp (); + + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + } + + /* Update CERT data that depends on SCAP. */ + update_cert_data (); + + g_info ("%s: Updating SCAP data aborted", __func__); + setproctitle ("Syncing SCAP: aborted"); +} + /** * @brief Try load the feed from feed CSV files. * @@ -3661,7 +3711,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cpes () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } @@ -3670,7 +3720,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cves () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } From 19d6be26bbabdd6544b4d387808a586653d0c93a Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 6 Mar 2024 08:12:43 +0100 Subject: [PATCH 28/46] CI: Push gvmd-build image to docker registry The workflow for building the gvmd-build image now has the option to push the image to the docker registry set. --- .github/workflows/build-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 7a6175c58..6e5802d8e 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -54,6 +54,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + push: true build-args: | GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }} file: .docker/build.Dockerfile From 90ea769216f858cd1ccd86a1437a77d493fe974b Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 6 Mar 2024 09:18:48 +0100 Subject: [PATCH 29/46] CI: Use ghcr.io for gvmd-build image The gvmd-build Docker image is now pushed to and pulled from ghcr.io. This is done because the image is meant to be used only in the gvmd building and testing workflows. --- .docker/prod.Dockerfile | 2 +- .github/workflows/build-and-test.yml | 4 ++-- .github/workflows/build-container.yml | 9 +++++---- .github/workflows/build-docs.yml | 2 +- .github/workflows/codeql-analysis-c.yml | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 212ecde50..642b02bc7 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -2,7 +2,7 @@ ARG VERSION=edge ARG GVM_LIBS_VERSION=oldstable ARG DEBIAN_FRONTEND=noninteractive -FROM greenbone/gvmd-build:${VERSION} as builder +FROM ghcr.io/greenbone/gvmd-build:${VERSION} as builder COPY . /source WORKDIR /source diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ce79fbfe8..158ad3733 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -22,7 +22,7 @@ jobs: scan-build: name: scan-build (clang static analyzer) runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: gchr.io/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: test-units: name: Unit Tests runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: gchr.io/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 6e5802d8e..0f76eb86a 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -23,7 +23,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ github.repository }}-build + images: gchr.io/${{ github.repository }}-build labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs @@ -40,11 +40,12 @@ jobs: else echo "gvm-libs-version=oldstable-edge" >> $GITHUB_OUTPUT fi - - name: Login to DockerHub + - name: Login to GitHub Docker registry uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ secrets.GREENBONE_BOT }} + password: ${{ secrets.GREENBONE_BOT_PACKAGES_WRITE_TOKEN }} - run: echo "Build and push ${{ steps.meta.outputs.tags }}" - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index f5901f219..5bd27b0d8 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -16,7 +16,7 @@ jobs: build-gmp-doc: name: Build GMP documentation runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: gchr.io/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql-analysis-c.yml index 42f53118a..f2e255696 100644 --- a/.github/workflows/codeql-analysis-c.yml +++ b/.github/workflows/codeql-analysis-c.yml @@ -19,7 +19,7 @@ jobs: actions: read contents: read security-events: write - container: greenbone/gvmd-build:stable + container: gchr.io/greenbone/gvmd-build:stable strategy: fail-fast: false From 74d38080968c6b44869f5d84337b1d7aa01b303b Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 6 Mar 2024 10:11:31 +0100 Subject: [PATCH 30/46] Use IMAGE_REGISTRY var for build image registry The registry for the build image is no longer hard-coded but using the variable IMAGE_REGISTRY. --- .docker/prod.Dockerfile | 3 ++- .github/workflows/build-and-test.yml | 4 ++-- .github/workflows/build-container.yml | 2 +- .github/workflows/build-docs.yml | 2 +- .github/workflows/codeql-analysis-c.yml | 2 +- .github/workflows/container.yml | 1 + 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 642b02bc7..47963c574 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -1,8 +1,9 @@ ARG VERSION=edge ARG GVM_LIBS_VERSION=oldstable ARG DEBIAN_FRONTEND=noninteractive +ARG IMAGE_REGISTRY=ghcr.io -FROM ghcr.io/greenbone/gvmd-build:${VERSION} as builder +FROM ${IMAGE_REGISTRY}/greenbone/gvmd-build:${VERSION} as builder COPY . /source WORKDIR /source diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 158ad3733..175d4a263 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -22,7 +22,7 @@ jobs: scan-build: name: scan-build (clang static analyzer) runs-on: ubuntu-latest - container: gchr.io/greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: test-units: name: Unit Tests runs-on: ubuntu-latest - container: gchr.io/greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 0f76eb86a..b927fd34d 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -23,7 +23,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: gchr.io/${{ github.repository }}-build + images: ${{ vars.IMAGE_REGISTRY }}/${{ github.repository }}-build labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 5bd27b0d8..6557c2ab6 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -16,7 +16,7 @@ jobs: build-gmp-doc: name: Build GMP documentation runs-on: ubuntu-latest - container: gchr.io/greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql-analysis-c.yml index f2e255696..add9937b5 100644 --- a/.github/workflows/codeql-analysis-c.yml +++ b/.github/workflows/codeql-analysis-c.yml @@ -19,7 +19,7 @@ jobs: actions: read contents: read security-events: write - container: gchr.io/greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable strategy: fail-fast: false diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 78a688f8d..7b534424d 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -71,6 +71,7 @@ jobs: build-args: | VERSION=${{ steps.container-opts.outputs.version }} GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }} + IMAGE_REGISTRY=${{ vars.IMAGE_REGISTRY }} file: .docker/prod.Dockerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} From fe0cfe7b39cec141431b977e48067dc7bcfb9394 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Fri, 1 Mar 2024 11:01:12 +0100 Subject: [PATCH 31/46] Change: Replace CPE xml_split with XML iterator When updating the CPEs, the CPE dictionary file is no longer split into separate temporary files. Instead a new XML file iterator added to gvm-libs is used to avoid creating the DOM of the whole file. Also, the buffered insert statements are now also run when a size threshold configurable via a setting is reached. This removes the dependency on the xml_split script, avoids problems with the temp split XML files being left behind in case of crashes and gives more control over the memory usage of the CPEs update. --- src/manage.h | 3 + src/manage_sql.c | 53 +++++- src/manage_sql.h | 4 + src/manage_sql_secinfo.c | 349 ++++++++++++++++----------------------- 4 files changed, 194 insertions(+), 215 deletions(-) diff --git a/src/manage.h b/src/manage.h index 9a73dfdcb..30e0642bb 100644 --- a/src/manage.h +++ b/src/manage.h @@ -3449,6 +3449,9 @@ setting_iterator_comment (iterator_t*); const char* setting_iterator_value (iterator_t*); +int +setting_value_int (const char *, int *); + int modify_setting (const gchar *, const gchar *, const gchar *, gchar **); diff --git a/src/manage_sql.c b/src/manage_sql.c index 2f2f8b7cd..210c242c3 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -328,9 +328,6 @@ static void set_credential_snmp_secret (credential_t, const char *, const char *, const char *); -static int -setting_value_int (const char *, int *); - static int setting_auto_cache_rebuild_int (); @@ -16005,6 +16002,19 @@ check_db_settings () " 'Delta Reports Version'," " 'Version of the generation of the Delta Reports.'," " '2' );"); + + if (sql_int ("SELECT count(*) FROM settings" + " WHERE uuid = '" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "'" + " AND " ACL_IS_GLOBAL () ";") + == 0) + sql ("INSERT into settings (uuid, owner, name, comment, value)" + " VALUES" + " ('" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "', NULL," + " 'SecInfo SQL Buffer Threshold'," + " 'Buffer size threshold in MiB for running buffered SQL statements'" + " || ' in SecInfo updates before the end of the file'" + " || ' being processed.'," + " '100' );"); } /** @@ -51978,7 +51988,7 @@ setting_value (const char *uuid, char **value) * * @return 0 success, -1 error. */ -static int +int setting_value_int (const char *uuid, int *value) { gchar *quoted_uuid; @@ -52683,6 +52693,8 @@ setting_name (const gchar *uuid) return "Feed Import Roles"; if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) return "Delta Reports Version"; + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + return "SecInfo SQL Buffer Threshold"; return NULL; } @@ -52722,12 +52734,15 @@ setting_description (const gchar *uuid) return "Roles given access to new resources from feed."; if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) return "Version of the generation of the Delta Reports."; + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + return "Buffer size threshold in MiB for running buffered SQL statements" + " in SecInfo updates before the end of the file being processed."; return NULL; } /** - * @brief Get the name of a setting. + * @brief Verify the value of a setting. * * @param[in] uuid UUID of setting. * @param[in] value Value of setting, to verify. @@ -52815,6 +52830,19 @@ setting_verify (const gchar *uuid, const gchar *value, const gchar *user) return 1; } + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD)) + { + int threshold; + threshold = atoi (value); + if (user) + { + if (threshold < 0 || threshold > (INT_MAX / 1048576)) + return 1; + } + else if (threshold < 0) + return 1; + } + return 0; } @@ -52870,6 +52898,15 @@ setting_normalise (const gchar *uuid, const gchar *value) return g_string_free (normalised, FALSE); } + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + { + int threshold; + threshold = atoi (value); + if (threshold < 0) + return NULL; + return g_strdup_printf ("%i", threshold); + } + return g_strdup (value); } @@ -52900,7 +52937,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database, && strcmp (uuid, SETTING_UUID_LSC_DEB_MAINTAINER) && strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER) && strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES) - && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION)) + && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) + && strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD)) { fprintf (stderr, "Error in setting UUID.\n"); return 3; @@ -52927,7 +52965,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database, if ((strcmp (uuid, SETTING_UUID_DEFAULT_CA_CERT) == 0) || (strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER) == 0) || (strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES) == 0) - || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0)) + || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) + || (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0)) { sql_rollback (); fprintf (stderr, diff --git a/src/manage_sql.h b/src/manage_sql.h index 7d8a1e4df..9f15f1269 100644 --- a/src/manage_sql.h +++ b/src/manage_sql.h @@ -142,6 +142,10 @@ */ #define SETTING_UUID_DELTA_REPORTS_VERSION "985a0c05-2140-4e66-9989-ce9a0906a5a9" +/** + * @brief UUID of 'SecInfo SQL Buffer Threshold' setting. + */ +#define SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "316275a9-3629-49ad-9cea-5b3ab155b93f" /** * @brief Trust constant for error. diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 4911b73d8..e4bc4d843 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -51,6 +51,7 @@ #include #include #include +#include #undef G_LOG_DOMAIN /** @@ -62,7 +63,7 @@ /* Static variables. */ /** - * @brief Maximum number of rows in an INSERT. + * @brief Maximum number of rows in a CPEs INSERT. */ #define CPE_MAX_CHUNK_SIZE 10000 @@ -185,130 +186,21 @@ increment_transaction_size (int* current_size) } } +/* Helper: buffer structure for INSERTs. */ + /** - * @brief Split a file. - * - * @param[in] path Path to file. - * @param[in] size Approx size of split files. In same format that - * xml_split accepts, eg "200Kb". - * @param[in] tail Text to replace last line of split files. - * - * @return Temp dir holding split files. + * @brief Get the SQL buffer size threshold converted from MiB to bytes. */ -static const gchar * -split_xml_file (gchar *path, const gchar *size, const gchar *tail) +int +setting_secinfo_sql_buffer_threshold_bytes () { - int ret; - static gchar dir[] = "/tmp/gvmd-split-xml-file-XXXXXX"; - gchar *previous_dir, *command; - - if (mkdtemp (dir) == NULL) - { - g_warning ("%s: Failed to make temp dir: %s", - __func__, - strerror (errno)); - return NULL; - } - - previous_dir = getcwd (NULL, 0); - if (previous_dir == NULL) - { - g_warning ("%s: Failed to getcwd: %s", - __func__, - strerror (errno)); - return NULL; - } + int threshold; - if (chdir (dir)) - { - g_warning ("%s: Failed to chdir: %s", - __func__, - strerror (errno)); - g_free (previous_dir); - return NULL; - } - - if (gvm_file_copy (path, "split.xml") == FALSE) - { - g_free (previous_dir); - return NULL; - } - - /* xml_split will chop split.xml into files that are roughly 'size' big. - * - * The generated files are always put in the directory that holds - * split.xml, as follows: - * - * split.xml Source XML. - * split-00.xml Master generated XML. No content, just includes other - * files. The include statements are wrapped in the - * root element from split.xml. - * split-01.xml Generated XML content. Wrapped in an - * element. - * split-02.xml Second generated content file. - * ... - * split-112.xml Last content, for example. - * - * Parsing the generated files independently will only work if the files - * contain the original root element (for example, because the parser - * requires the namespace definitions to be present). - * - * So the command below needs to mess around a little bit to replace the - * wrapper XML element in split-01.xml, split-02.xml, etc with the root - * element from split-00.xml. - * - * Using tail and head is not super robust, but it's simple and it will - * work as long as xml_split keeps the opening of the wrapper element - * in split-00.xml on a dedicated line. (It doesn't do this for the - * closing element, so we use the tail argument instead.) - */ - - command = g_strdup_printf - ("xml_split -s%s split.xml" - " && head -n 2 split-00.xml > head.xml" - " && echo '%s' > tail.xml" - " && for F in split-*.xml; do" - /* Remove the first two lines and last line. */ - " awk 'NR>3 {print last} {last=$0}' $F > body.xml" - /* Combine with new start and end. */ - " && cat head.xml body.xml tail.xml > $F;" - " done", - size, - tail); - - g_debug ("%s: command: %s", __func__, command); - ret = system (command); - if ((ret == -1) || WIFEXITED(ret) == 0 || WEXITSTATUS (ret)) - { - g_warning ("%s: system failed with ret %i, %i (%i), %s", - __func__, - ret, - WIFEXITED (ret), - WIFEXITED (ret) ? WEXITSTATUS (ret) : 0, - command); - g_free (command); - - if (chdir (previous_dir)) - g_warning ("%s: and failed to chdir back", __func__); - g_free (previous_dir); - - return NULL; - } - - g_free (command); + setting_value_int (SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD, &threshold); - if (chdir (previous_dir)) - g_warning ("%s: Failed to chdir back (will continue anyway)", - __func__); - - g_free (previous_dir); - - return dir; + return threshold * 1048576; } - -/* Helper: buffer structure for INSERTs. */ - /** * @brief Buffer for INSERT statements. */ @@ -316,26 +208,34 @@ typedef struct { array_t *statements; ///< Buffered statements. GString *statement; ///< Current statement. + int statements_size; ///< Sum of lengths of all statements buffered. + int max_statements_size; ///< Auto-run at this statement_size, 0 for never. int current_chunk_size; ///< Number of rows in current statement. int max_chunk_size; ///< Max number of rows per INSERT. gchar *open_sql; ///< SQL to open each statement. gchar *close_sql; ///< SQL to close each statement. } inserts_t; +static void +inserts_run (inserts_t *, gboolean); + /** * @brief Check size of current statement. * * @param[in] inserts Insert buffer. - * @param[in] max_chunk_size Max chunk size. + * @param[in] max_chunk_size Max chunk size per statement. + * @param[in] max_statements_size Automatically run at this statements size. * @param[in] open_sql SQL to to start each statement. * @param[in] close_sql SQL to append to the end of each statement. */ static void -inserts_init (inserts_t *inserts, int max_chunk_size, const gchar *open_sql, - const gchar *close_sql) +inserts_init (inserts_t *inserts, int max_chunk_size, int max_statements_size, + const gchar *open_sql, const gchar *close_sql) { inserts->statements = make_array (); inserts->statement = NULL; + inserts->statements_size = 0; + inserts->max_statements_size = max_statements_size; inserts->current_chunk_size = 0; inserts->max_chunk_size = max_chunk_size; inserts->open_sql = open_sql ? g_strdup (open_sql) : NULL; @@ -377,8 +277,15 @@ inserts_check_size (inserts_t *inserts) { inserts_statement_close (inserts); array_add (inserts->statements, inserts->statement); + inserts->statements_size += inserts->statement->len; inserts->statement = NULL; inserts->current_chunk_size = 0; + + if (inserts->max_statements_size + && inserts-> statements_size >= inserts->max_statements_size) + { + inserts_run (inserts, FALSE); + } } if (inserts->statement == NULL) @@ -392,7 +299,26 @@ inserts_check_size (inserts_t *inserts) } /** - * @brief Free everything. + * @brief Free only the statements in an inserts buffer so it can be reused. + * + * @param[in] inserts Insert buffer. + */ +static void +inserts_free_statements (inserts_t *inserts) +{ + int index; + + for (index = 0; index < inserts->statements->len; index++) + { + g_string_free (g_ptr_array_index (inserts->statements, index), TRUE); + inserts->statements->pdata[index] = NULL; + } + g_ptr_array_set_size (inserts->statements, 0); + inserts->statements_size = 0; +} + +/** + * @brief Free all fields in an inserts buffer. * * @param[in] inserts Insert buffer. */ @@ -413,9 +339,10 @@ inserts_free (inserts_t *inserts) * @brief Run the INSERT SQL, freeing the buffers. * * @param[in] inserts Insert buffer. + * @param[in] finalize Whether to free the whole inserts buffer afterwards. */ static void -inserts_run (inserts_t *inserts) +inserts_run (inserts_t *inserts, gboolean finalize) { guint index; @@ -435,7 +362,10 @@ inserts_run (inserts_t *inserts) sql ("%s", statement->str); } - inserts_free (inserts); + if (finalize) + inserts_free (inserts); + else + inserts_free_statements (inserts); } @@ -2119,45 +2049,33 @@ insert_scap_cpe_details (inserts_t *inserts, element_t cpe_item) static int update_scap_cpes_from_file (const gchar *path) { - GError *error; - element_t element, cpe_list, cpe_item; - gchar *xml; - gsize xml_len; - inserts_t inserts, details_inserts; - - g_debug ("%s: parsing %s", __func__, path); - - error = NULL; - g_file_get_contents (path, &xml, &xml_len, &error); - if (error) - { - g_warning ("%s: Failed to get contents: %s", - __func__, - error->message); - g_error_free (error); - return -1; - } - - if (parse_element (xml, &element)) - { - g_free (xml); - g_warning ("%s: Failed to parse element", __func__); - return -1; - } - g_free (xml); + int ret; + element_t cpe_item; + inserts_t inserts; + xml_file_iterator_t file_iterator; + gchar *error_message = NULL; - cpe_list = element; - if (strcmp (element_name (cpe_list), "cpe-list")) + file_iterator = xml_file_iterator_new (); + ret = xml_file_iterator_init_from_file_path (file_iterator, path, 1); + switch (ret) { - element_free (element); - g_warning ("%s: CPE dictionary missing CPE-LIST", __func__); - return -1; + case 0: + break; + case 2: + g_warning ("%s: Could not open file '%s' for XML file iterator: %s", + __func__, path, strerror(errno)); + return -1; + default: + g_warning ("%s: Could not initialize XML file iterator", + __func__); + return -1; } sql_begin_immediate (); inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpes" " (uuid, name, title, creation_time," " modification_time, status, deprecated_by_id," @@ -2171,7 +2089,15 @@ update_scap_cpes_from_file (const gchar *path) " status = EXCLUDED.status," " deprecated_by_id = EXCLUDED.deprecated_by_id," " nvd_id = EXCLUDED.nvd_id"); - cpe_item = element_first_child (cpe_list); + + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get first CPE XML element: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } while (cpe_item) { gchar *modification_date; @@ -2180,7 +2106,15 @@ update_scap_cpes_from_file (const gchar *path) if (strcmp (element_name (cpe_item), "cpe-item")) { - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } continue; } @@ -2205,47 +2139,82 @@ update_scap_cpes_from_file (const gchar *path) if (insert_scap_cpe (&inserts, cpe_item, item_metadata, modification_time)) goto fail; - cpe_item = element_next (cpe_item); + + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element: %s", + __func__, error_message); + g_free (error_message); + error_message = NULL; + } } - inserts_run (&inserts); + inserts_run (&inserts, TRUE); sql_commit (); + xml_file_iterator_rewind (file_iterator); + // Extract and save details XML. - inserts_init (&details_inserts, + inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpe_details" " (cpe_id, details_xml)" " VALUES", " ON CONFLICT (cpe_id) DO UPDATE" " SET details_xml = EXCLUDED.details_xml"); - cpe_item = element_first_child (cpe_list); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get first CPE XML element for details: %s", + __func__, error_message); + g_free (error_message); + error_message = NULL; + } while (cpe_item) { if (strcmp (element_name (cpe_item), "cpe-item")) { - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element" + " for details: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } continue; } - if (insert_scap_cpe_details (&details_inserts, cpe_item)) + if (insert_scap_cpe_details (&inserts, cpe_item)) goto fail; - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element for details: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } } - element_free (element); - sql_begin_immediate(); - inserts_run (&details_inserts); + inserts_run (&inserts, TRUE); sql_commit(); + xml_file_iterator_free (file_iterator); return 0; fail: inserts_free (&inserts); - element_free (element); g_warning ("Update of CPEs failed"); sql_commit (); + xml_file_iterator_free (file_iterator); return -1; } @@ -2258,9 +2227,8 @@ static int update_scap_cpes () { gchar *full_path; - const gchar *split_dir; GStatBuf state; - int index; + int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, "official-cpe-dictionary_v2.2.xml", @@ -2277,44 +2245,9 @@ update_scap_cpes () g_info ("Updating CPEs"); - split_dir = split_xml_file (full_path, "40Mb", ""); - if (split_dir == NULL) - { - int ret; - - g_warning ("%s: Failed to split CPEs, attempting with full file", - __func__); - ret = update_scap_cpes_from_file (full_path); - g_free (full_path); - return ret; - } - g_free (full_path); - - for (index = 1; 1; index++) - { - int ret; - gchar *path, *name; - - name = g_strdup_printf ("split-%02i.xml", index); - path = g_build_filename (split_dir, name, NULL); - g_free (name); - - if (g_stat (path, &state)) - { - g_free (path); - break; - } - - ret = update_scap_cpes_from_file (path); - g_free (path); - if (ret < 0) - { - gvm_file_remove_recurse (split_dir); - return -1; - } - } - - gvm_file_remove_recurse (split_dir); + ret = update_scap_cpes_from_file (full_path); + if (ret) + return -1; return 0; } From 18d4acd02e0ac4336db9872f27f10cd816a2f256 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Tue, 5 Mar 2024 15:23:28 +0100 Subject: [PATCH 32/46] Address review comments for new XML file iterator The error handling is improved by freeing the iterator on errors, consistently jumping to the failure handling and handling cases where (re-)initalizing the parser context fails. Also the limit for the threshold is now always applied. --- src/manage_sql.c | 7 +------ src/manage_sql_secinfo.c | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 210c242c3..c3b11ea68 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -52834,12 +52834,7 @@ setting_verify (const gchar *uuid, const gchar *value, const gchar *user) { int threshold; threshold = atoi (value); - if (user) - { - if (threshold < 0 || threshold > (INT_MAX / 1048576)) - return 1; - } - else if (threshold < 0) + if (threshold < 0 || threshold > (INT_MAX / 1048576)) return 1; } diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index e4bc4d843..bf1f26aa1 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2064,10 +2064,17 @@ update_scap_cpes_from_file (const gchar *path) case 2: g_warning ("%s: Could not open file '%s' for XML file iterator: %s", __func__, path, strerror(errno)); + xml_file_iterator_free (file_iterator); + return -1; + case 3: + g_warning ("%s: Could not create parser context for XML file iterator", + __func__); + xml_file_iterator_free (file_iterator); return -1; default: g_warning ("%s: Could not initialize XML file iterator", __func__); + xml_file_iterator_free (file_iterator); return -1; } @@ -2148,13 +2155,21 @@ update_scap_cpes_from_file (const gchar *path) __func__, error_message); g_free (error_message); error_message = NULL; + goto fail; } } inserts_run (&inserts, TRUE); sql_commit (); + sql_begin_immediate(); - xml_file_iterator_rewind (file_iterator); + if (xml_file_iterator_rewind (file_iterator)) + { + g_warning ("%s: Could not create parser context for XML file iterator" + " for details.", + __func__); + goto fail; + } // Extract and save details XML. inserts_init (&inserts, @@ -2172,6 +2187,7 @@ update_scap_cpes_from_file (const gchar *path) __func__, error_message); g_free (error_message); error_message = NULL; + goto fail; } while (cpe_item) { @@ -2203,7 +2219,6 @@ update_scap_cpes_from_file (const gchar *path) } } - sql_begin_immediate(); inserts_run (&inserts, TRUE); sql_commit(); xml_file_iterator_free (file_iterator); From b0b1dee2ce4a9566d2d1aab6365a029e3b3a6db7 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 6 Mar 2024 10:52:34 +0100 Subject: [PATCH 33/46] Remove: Remove xml_split / xml-twig-tools dependency This optional dependency is no longer needed with the new XML file iterator. --- .docker/prod.Dockerfile | 1 - INSTALL.md | 3 --- src/CMakeLists.txt | 8 -------- 3 files changed, 12 deletions(-) diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 47963c574..531cbe555 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -100,7 +100,6 @@ RUN apt-get update && \ texlive-fonts-recommended \ texlive-latex-extra \ wget \ - xml-twig-tools \ xmlstarlet \ xsltproc \ zip && \ diff --git a/INSTALL.md b/INSTALL.md index 5b743f248..ac327bc57 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -705,9 +705,6 @@ Prerequisites for S/MIME support (e.g. email encryption): Prerequisites for certificate generation: * GnuTLS certtool (Debian package: gnutls-bin) -Prerequisites (recommended) to lower sync RAM usage -* xml_split (Debian package: xml-twig-tools) - ## Static code analysis with the Clang Static Analyzer If you want to use the Clang Static Analyzer (https://clang-analyzer.llvm.org/) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a81d9286..e21f17e38 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,14 +55,6 @@ elseif ((CMAKE_MATCH_1 EQUAL 9 AND CMAKE_MATCH_2 LESS 6) message (STATUS "PostgreSQL version ${CMAKE_MATCH_1}.${CMAKE_MATCH_2}${CMAKE_MATCH_3}") endif (NOT CMAKE_MATCH_1) -message (STATUS "Looking for xml_split...") -find_program (XML_SPLIT_EXECUTABLE xml_split DOC "xml_split") -if (NOT XML_SPLIT_EXECUTABLE) - message (WARNING "xml_split is recommended to reduce SCAP sync memory usage (Debian package xml-twig-tools).") -else (NOT XML_SPLIT_EXECUTABLE) - message (STATUS "Looking for xml_split... ${XML_SPLIT_EXECUTABLE}") -endif (NOT XML_SPLIT_EXECUTABLE) - message (STATUS "Looking for xsltproc...") find_program (XSLTPROC_EXECUTABLE xsltproc DOC "xsltproc") if (NOT XSLTPROC_EXECUTABLE) From 8891f74c3a61f61af2eb9ee2948ef078f5a9849c Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 6 Mar 2024 11:00:15 +0100 Subject: [PATCH 34/46] Raise required gvm-libs version to 22.9 --- src/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e21f17e38..c2f3aebb0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,10 +26,10 @@ find_package (Threads) ## list and throw an error, otherwise long install-cmake-install-cmake cycles ## might occur. -pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.8) -pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.8) -pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.8) -pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.8) +pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.9) +pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.9) +pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.9) +pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.9) pkg_check_modules (GNUTLS REQUIRED gnutls>=3.2.15) pkg_check_modules (GLIB REQUIRED glib-2.0>=2.42) pkg_check_modules (LIBBSD REQUIRED libbsd) From 3a1ee4c813d88ba7623dfdc3af76a10dc8e5ba5e Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Tue, 5 Mar 2024 10:39:48 +0100 Subject: [PATCH 35/46] Change: Use XML file iterator for CVE updates The update of the CVEs now also uses the XML file iterator instead of loading and parsing the whole file. Also, a memory leak in insert_cve_products has been fixed. This reduces the amount of memory used during the update. --- src/manage_sql_secinfo.c | 80 +++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index bf1f26aa1..13d54605d 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2370,8 +2370,7 @@ insert_cve_products (element_t list, resource_t cve, gchar *quoted_product, *product_decoded; gchar *product_tilde; - product_decoded = g_uri_unescape_string - (element_text (product), NULL); + product_decoded = g_uri_unescape_string (product_text, NULL); product_tilde = string_replace (product_decoded, "~", "%7E", "%7e", NULL); @@ -2641,12 +2640,13 @@ insert_cve_from_entry (element_t entry, element_t last_modified, static int update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) { - GError *error; - element_t element, entry; - gchar *xml, *full_path; - gsize xml_len; + gchar *error_message = NULL; + xml_file_iterator_t iterator; + element_t entry; + gchar *full_path; GStatBuf state; int transaction_size = 0; + int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, xml_path, NULL); @@ -2660,29 +2660,31 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) g_info ("Updating %s", full_path); - error = NULL; - g_file_get_contents (full_path, &xml, &xml_len, &error); - if (error) - { - g_warning ("%s: Failed to get contents: %s", - __func__, - error->message); - g_error_free (error); - g_free (full_path); - return -1; - } - - if (parse_element (xml, &element)) + iterator = xml_file_iterator_new (); + ret = xml_file_iterator_init_from_file_path (iterator, full_path, 1); + switch (ret) { - g_free (xml); - g_warning ("%s: Failed to parse element", __func__); - g_free (full_path); - return -1; + case 0: + break; + case 2: + g_warning ("%s: Could not open file '%s' for XML file iterator: %s", + __func__, full_path, strerror(errno)); + xml_file_iterator_free (iterator); + return -1; + case 3: + g_warning ("%s: Could not create parser context for XML file iterator", + __func__); + xml_file_iterator_free (iterator); + return -1; + default: + g_warning ("%s: Could not initialize XML file iterator", + __func__); + xml_file_iterator_free (iterator); + return -1; } - g_free (xml); sql_begin_immediate (); - entry = element_first_child (element); + entry = xml_file_iterator_next (iterator, &error_message); while (entry) { if (strcmp (element_name (entry), "entry") == 0) @@ -2692,28 +2694,40 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) last_modified = element_child (entry, "vuln:last-modified-datetime"); if (last_modified == NULL) { - g_warning ("%s: vuln:last-modified-datetime missing", - __func__); + error_message = g_strdup ("vuln:last-modified-datetime missing"); goto fail; } if (insert_cve_from_entry (entry, last_modified, hashed_cpes, &transaction_size)) - goto fail; + { + error_message = g_strdup ("Insert of CVE into database failed"); + goto fail; + } } - entry = element_next (entry); + + element_free (entry); + entry = xml_file_iterator_next (iterator, &error_message); } - element_free (element); + if (error_message) + goto fail; + + xml_file_iterator_free (iterator); g_free (full_path); sql_commit (); return 0; fail: - element_free (element); - g_warning ("Update of CVEs failed at file '%s'", - full_path); + xml_file_iterator_free (iterator); + if (error_message) + g_warning ("Update of CVEs failed at file '%s': %s", + full_path, error_message); + else + g_warning ("Update of CVEs failed at file '%s'", + full_path); g_free (full_path); + g_free (error_message); sql_commit (); return -1; } From 894a7197ff871505093cc0da35e44fa77ae69e79 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 5 Mar 2024 15:18:36 +0100 Subject: [PATCH 36/46] Fix: Handle value of last_update and SCAP schema when feed update fails. --- src/manage_sql_secinfo.c | 108 ++++++++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index bf1f26aa1..3e68d2b8a 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3375,38 +3375,15 @@ update_scap_placeholders () " WHERE cpe=cpes.id))" " WHERE cpes.title IS NULL;"); } - + /** - * @brief Finish scap update. - * - * @return 0 success, -1 error. + * @brief Update CERT data that depends on SCAP. */ -static int -update_scap_end () +static void +update_cert_data () { int cert_db_version; - g_debug ("%s: update timestamp", __func__); - - update_scap_timestamp (); - - /* Replace the real scap schema with the new one. */ - - if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" - " information_schema.schemata" - " WHERE schema_name = 'scap');")) - { - sql ("ALTER SCHEMA scap RENAME TO scap3;"); - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - sql ("DROP SCHEMA scap3 CASCADE;"); - /* View 'vulns' contains references into the SCAP schema, so it is - * removed by the CASCADE. */ - create_view_vulns (); - } - else - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - - /* Update CERT data that depends on SCAP. */ cert_db_version = manage_cert_db_version(); if (cert_db_version == -1) @@ -3438,6 +3415,39 @@ update_scap_end () update_cvss_dfn_cert (1, last_cert_update, last_scap_update); update_cvss_cert_bund (1, last_cert_update, last_scap_update); } +} + +/** + * @brief Finish scap update. + * + * @return 0 success, -1 error. + */ +static int +update_scap_end () +{ + g_debug ("%s: update timestamp", __func__); + + update_scap_timestamp (); + + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + + /* Update CERT data that depends on SCAP. */ + + update_cert_data (); /* Analyze. */ @@ -3451,6 +3461,46 @@ update_scap_end () return 0; } +/** + * @brief Abort scap update. + */ +static void +abort_scap_update () +{ + g_debug ("%s: update timestamp", __func__); + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + update_scap_timestamp (); + sql ("UPDATE scap.meta SET value = " + " (SELECT value from scap2.meta WHERE name = 'last_update')" + " WHERE name = 'last_update';"); + sql ("DROP SCHEMA scap2 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + } + else + { + /* reset scap2 schema */ + manage_db_init ("scap"); + manage_db_init_indexes ("scap"); + manage_db_add_constraints ("scap"); + + update_scap_timestamp (); + + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + } + + /* Update CERT data that depends on SCAP. */ + update_cert_data (); + + g_info ("%s: Updating SCAP data aborted", __func__); + setproctitle ("Syncing SCAP: aborted"); +} + /** * @brief Try load the feed from feed CSV files. * @@ -3609,7 +3659,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cpes () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } @@ -3618,7 +3668,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cves () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } From 1aea5e646fa726ce4bdc9c6642f14ac4daffd0e6 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Thu, 7 Mar 2024 11:47:53 +0100 Subject: [PATCH 37/46] Some improvements. --- src/manage_sql_secinfo.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index bfe591db2..d45fc11df 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3433,10 +3433,8 @@ update_cert_data () /** * @brief Finish scap update. - * - * @return 0 success, -1 error. */ -static int +static void update_scap_end () { g_debug ("%s: update timestamp", __func__); @@ -3471,8 +3469,6 @@ update_scap_end () g_info ("%s: Updating SCAP info succeeded", __func__); setproctitle ("Syncing SCAP: done"); - - return 0; } /** @@ -3499,9 +3495,21 @@ abort_scap_update () else { /* reset scap2 schema */ - manage_db_init ("scap"); - manage_db_init_indexes ("scap"); - manage_db_add_constraints ("scap"); + if (manage_db_init ("scap")) + { + g_warning ("%s: could not reset scap2 schema, db init failed", __func__); + return; + } + if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not reset scap2 schema, init indexes failed", __func__); + return; + } + if (manage_db_add_constraints ("scap")) + { + g_warning ("%s: could not reset scap2 schema, add constrains failed", __func__); + return; + } update_scap_timestamp (); @@ -3574,7 +3582,8 @@ try_load_csv () return -1; } - return update_scap_end (); + update_scap_end (); + return 0; } return 1; } @@ -3701,7 +3710,8 @@ update_scap (gboolean reset_scap_db) update_scap_placeholders (); - return update_scap_end (); + update_scap_end (); + return 0; } /** From f283ea20018bd15f6d32baea309d9cddc89cace6 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Thu, 7 Mar 2024 09:11:59 +0100 Subject: [PATCH 38/46] Handle alert_filter_get with and without a filter selected. When an alert with compose data but with no filter selected was added to a task, the task was set to interrupted after it reached a progress of 100 % due to a not allocated alert_filter_get struct. --- src/manage_sql.c | 81 ++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index c3b11ea68..7f5182670 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -12139,25 +12139,21 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data, if (filter_return) *filter_return = filter; + (*alert_filter_get) = g_malloc0 (sizeof (get_data_t)); + (*alert_filter_get)->details = base_get_data->details; + (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination; + (*alert_filter_get)->ignore_max_rows_per_page + = base_get_data->ignore_max_rows_per_page; + if (filter) { - (*alert_filter_get) = g_malloc0 (sizeof (get_data_t)); - (*alert_filter_get)->details = base_get_data->details; - (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination; - (*alert_filter_get)->ignore_max_rows_per_page - = base_get_data->ignore_max_rows_per_page; (*alert_filter_get)->filt_id = g_strdup (filt_id); (*alert_filter_get)->filter = filter_term (filt_id); } else - (*alert_filter_get) = NULL; - - ignore_pagination = alert_data (alert, "method", - "composer_ignore_pagination"); - if (ignore_pagination) { - (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination); - g_free (ignore_pagination); + (*alert_filter_get)->filt_id = NULL; + (*alert_filter_get)->filter = g_strdup(""); } /* Adjust filter for report composer. @@ -12168,39 +12164,44 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data, * We simply use these fields to adjust the filter. In the future we'll * remove the filter terms and extend the way we get the report. */ - if (filter) + gchar *include_notes, *include_overrides; + + ignore_pagination = alert_data (alert, "method", + "composer_ignore_pagination"); + if (ignore_pagination) { - gchar *include_notes, *include_overrides; + (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination); + g_free (ignore_pagination); + } - include_notes = alert_data (alert, "method", - "composer_include_notes"); - if (include_notes) - { - gchar *new_filter; + include_notes = alert_data (alert, "method", + "composer_include_notes"); + if (include_notes) + { + gchar *new_filter; - new_filter = g_strdup_printf ("notes=%i %s", - atoi (include_notes), - (*alert_filter_get)->filter); - g_free ((*alert_filter_get)->filter); - (*alert_filter_get)->filter = new_filter; - (*alert_filter_get)->filt_id = NULL; - g_free (include_notes); - } + new_filter = g_strdup_printf ("notes=%i %s", + atoi (include_notes), + (*alert_filter_get)->filter); + g_free ((*alert_filter_get)->filter); + (*alert_filter_get)->filter = new_filter; + (*alert_filter_get)->filt_id = NULL; + g_free (include_notes); + } - include_overrides = alert_data (alert, "method", - "composer_include_overrides"); - if (include_overrides) - { - gchar *new_filter; + include_overrides = alert_data (alert, "method", + "composer_include_overrides"); + if (include_overrides) + { + gchar *new_filter; - new_filter = g_strdup_printf ("overrides=%i %s", - atoi (include_overrides), - (*alert_filter_get)->filter); - g_free ((*alert_filter_get)->filter); - (*alert_filter_get)->filter = new_filter; - (*alert_filter_get)->filt_id = NULL; - g_free (include_overrides); - } + new_filter = g_strdup_printf ("overrides=%i %s", + atoi (include_overrides), + (*alert_filter_get)->filter); + g_free ((*alert_filter_get)->filter); + (*alert_filter_get)->filter = new_filter; + (*alert_filter_get)->filt_id = NULL; + g_free (include_overrides); } return 0; From bf8b7842e657ef70c5d5db0b7ba71af33dd6f79e Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Thu, 7 Mar 2024 15:54:03 +0100 Subject: [PATCH 39/46] One more improvement. --- src/manage_sql_secinfo.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d45fc11df..0a2d90cc7 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3491,6 +3491,8 @@ abort_scap_update () /* View 'vulns' contains references into the SCAP schema, so it is * removed by the CASCADE. */ create_view_vulns (); + /* Update CERT data that depends on SCAP. */ + update_cert_data (); } else { @@ -3498,27 +3500,27 @@ abort_scap_update () if (manage_db_init ("scap")) { g_warning ("%s: could not reset scap2 schema, db init failed", __func__); - return; } - if (manage_db_init_indexes ("scap")) + else if (manage_db_init_indexes ("scap")) { g_warning ("%s: could not reset scap2 schema, init indexes failed", __func__); - return; } - if (manage_db_add_constraints ("scap")) + else if (manage_db_add_constraints ("scap")) { g_warning ("%s: could not reset scap2 schema, add constrains failed", __func__); - return; } - update_scap_timestamp (); - - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap2');")) + { + update_scap_timestamp (); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + /* Update CERT data that depends on SCAP. */ + update_cert_data (); + } } - /* Update CERT data that depends on SCAP. */ - update_cert_data (); - g_info ("%s: Updating SCAP data aborted", __func__); setproctitle ("Syncing SCAP: aborted"); } From f660381733c6bca7b75dfcfdc6b4e27fefdaeb41 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Mon, 4 Mar 2024 13:52:42 +0100 Subject: [PATCH 40/46] Fix: Apply exclude_hosts of target in CVE scans CVE scans now exclude IP addresses specified in the exclude_hosts of the task target. This makes the behavior consistent with normal OpenVAS scans. --- src/manage.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/manage.c b/src/manage.c index e0ef3bb4b..d3d95483a 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3167,7 +3167,7 @@ static int fork_cve_scan_handler (task_t task, target_t target) { int pid; - char *report_id, *hosts; + char *report_id, *hosts, *exclude_hosts; gvm_hosts_t *gvm_hosts; gvm_host_t *gvm_host; @@ -3234,6 +3234,8 @@ fork_cve_scan_handler (task_t task, target_t target) exit (1); } + exclude_hosts = target_exclude_hosts (target); + reset_task (task); set_task_start_time_epoch (task, time (NULL)); set_scan_start_time_epoch (global_current_report, time (NULL)); @@ -3241,7 +3243,11 @@ fork_cve_scan_handler (task_t task, target_t target) /* Add the results. */ gvm_hosts = gvm_hosts_new (hosts); + gvm_hosts_exclude (gvm_hosts, exclude_hosts ?: ""); + free (hosts); + free (exclude_hosts); + while ((gvm_host = gvm_hosts_next (gvm_hosts))) if (cve_scan_host (task, global_current_report, gvm_host)) { From ccb000c8df89b8370f6d85a7e387ad1af6ecd91c Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Fri, 8 Mar 2024 08:07:43 +0100 Subject: [PATCH 41/46] Add error check when excluding hosts in CVE scan Co-authored-by: Ahmed <144101267+a-h-abdelsalam@users.noreply.github.com> --- src/manage.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/manage.c b/src/manage.c index d3d95483a..b239b0649 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3243,9 +3243,19 @@ fork_cve_scan_handler (task_t task, target_t target) /* Add the results. */ gvm_hosts = gvm_hosts_new (hosts); - gvm_hosts_exclude (gvm_hosts, exclude_hosts ?: ""); - free (hosts); + + if (gvm_hosts_exclude (gvm_hosts, exclude_hosts ?: "") < 0) + { + set_task_interrupted (task, + "Failed to exclude hosts." + " Interrupting scan."); + set_report_scan_run_status (global_current_report, TASK_STATUS_INTERRUPTED); + gvm_hosts_free (gvm_hosts); + free (exclude_hosts); + gvm_close_sentry (); + exit(1); + } free (exclude_hosts); while ((gvm_host = gvm_hosts_next (gvm_hosts))) From edc6448c08fd11768833f6e3164f7e96e339f5a8 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Fri, 8 Mar 2024 17:36:52 +0100 Subject: [PATCH 42/46] Fix: Improved the SQL for the selection of the delta report results. --- src/manage_sql.c | 50 ++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 7f5182670..93284576a 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -27930,23 +27930,41 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, "nvts_cols"); extra_with = g_strdup_printf(" comparison AS (" - " WITH r1 as (SELECT results.id, description, host, report, port," + " WITH r1a as (SELECT results.id, description, host, report, port," " severity, nvt, results.qod, results.uuid, hostname," " path, r1_lateral.new_severity as new_severity " " FROM results " - " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)" + " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)" " AS nvts_cols" " ON nvts_cols.nvts_oid = results.nvt" " %s, LATERAL %s AS r1_lateral" " WHERE report = %llu)," - " r2 as (SELECT results.*, r2_lateral.new_severity AS new_severity" + " r2a as (SELECT results.*, r2_lateral.new_severity AS new_severity" " FROM results" - " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)" + " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)" " AS nvts_cols" " ON nvts_cols.nvts_oid = results.nvt" " %s, LATERAL %s AS r2_lateral" - " WHERE report = %llu)" - " SELECT r1.id AS result1_id," + " WHERE report = %llu)," + " r1 as (SELECT DISTINCT ON (r1a.id) r1a.*, r2a.id as r2id, row_number() over w1 as r1_rank" + " FROM r1a LEFT JOIN r2a ON r1a.host = r2a.host" + " AND normalize_port(r1a.port) = normalize_port(r2a.port)" + " AND r1a.nvt = r2a.nvt " + " AND (r1a.new_severity = 0) = (r2a.new_severity = 0)" + " AND (r1a.description = r2a.description)" + " WINDOW w1 AS (PARTITION BY r1a.host, normalize_port(r1a.port)," + " r1a.nvt, r1a.new_severity = 0, r2a.id is null ORDER BY r2a.id)" + " ORDER BY r1a.id)," + " r2 as (SELECT DISTINCT ON (r2a.id) r2a.*, r1a.id as r1id, row_number() over w2 as r2_rank" + " FROM r2a LEFT JOIN r1a ON r2a.host = r1a.host" + " AND normalize_port(r2a.port) = normalize_port(r1a.port)" + " AND r2a.nvt = r1a.nvt " + " AND (r2a.new_severity = 0) = (r1a.new_severity = 0)" + " AND (r2a.description = r1a.description)" + " WINDOW w2 AS (PARTITION BY r2a.host, normalize_port(r2a.port)," + " r2a.nvt, r2a.new_severity = 0, r1a.id is null ORDER BY r1a.id)" + " ORDER BY r2a.id)" + " (SELECT r1.id AS result1_id," " r2.id AS result2_id," " compare_results(" " r1.description," @@ -27972,7 +27990,7 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, " r2.path AS delta_path," " r2.host AS delta_host," RESULT_HOSTNAME_SQL("r2.hostname", "r2.host", "r2.report") - " AS delta_hostname," + " AS delta_hostname," " r2.nvt_version AS delta_nvt_version" " FROM r1" " FULL OUTER JOIN r2" @@ -27980,20 +27998,10 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, " AND normalize_port(r1.port) = normalize_port(r2.port)" " AND r1.nvt = r2.nvt " " AND (r1.new_severity = 0) = (r2.new_severity = 0)" - " AND (r1.description = r2.description" - " OR NOT EXISTS (SELECT * FROM r2" - " WHERE r1.description = r2.description" - " AND r1.host = r2.host" - " AND normalize_port(r1.port) = normalize_port(r2.port)" - " AND r1.nvt = r2.nvt" - " AND (r1.new_severity = 0) = (r2.new_severity = 0))" - " OR NOT EXISTS (SELECT * FROM r1" - " WHERE r1.description = r2.description" - " AND r1.host = r2.host" - " AND normalize_port(r1.port) = normalize_port(r2.port)" - " AND r1.nvt = r2.nvt" - " AND (r1.new_severity = 0) = (r2.new_severity = 0)))" - " )", + " AND ((r1id IS NULL AND r2id IS NULL) OR" + " r2id = r2.id OR r1id = r1.id)" + " AND r1_rank = r2_rank" + " ) ) ", opts_tables, with_lateral, report, From 57a6d058863685c38e2f8cb0291eeaf798083822 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Mon, 11 Mar 2024 10:45:44 +0100 Subject: [PATCH 43/46] Fix: Set preference components when modifying config When modifying a config, the database fields containing the VT id, preference id, type and name are now set for all types. This fixes the preferences not working correctly after modifying them. --- src/manage_sql_configs.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/manage_sql_configs.c b/src/manage_sql_configs.c index 4fe5b0814..ae4b9454f 100644 --- a/src/manage_sql_configs.c +++ b/src/manage_sql_configs.c @@ -3627,6 +3627,13 @@ modify_config_preference (config_t config, const char* nvt, g_free (quoted_name); quoted_name = sql_quote (splits[3]); } + else + { + quoted_pref_nvt = sql_quote (splits[0]); + pref_id = atoi (splits[1]); + quoted_pref_type = sql_quote (splits[2]); + quoted_pref_name = sql_quote (splits[3]); + } } g_strfreev (splits); @@ -3639,12 +3646,25 @@ modify_config_preference (config_t config, const char* nvt, config, nvt ? "= 'PLUGINS_PREFS'" : "= 'SERVER_PREFS'", quoted_name); - sql ("INSERT INTO config_preferences" - " (config, type, name, value, pref_nvt, pref_id, pref_type, pref_name)" - " VALUES (%llu, %s, '%s', '%s', '%s', %i, '%s', '%s');", - config, nvt ? "'PLUGINS_PREFS'" : "'SERVER_PREFS'", quoted_name, - quoted_value, quoted_pref_nvt, pref_id, quoted_pref_type, - quoted_pref_name); + if (nvt) + { + sql ("INSERT INTO config_preferences" + " (config, type, name, value," + " pref_nvt, pref_id, pref_type, pref_name)" + " VALUES (%llu, 'PLUGINS_PREFS', '%s', '%s'," + " '%s', %i, '%s', '%s');", + config, quoted_name, quoted_value, + quoted_pref_nvt, pref_id, quoted_pref_type, quoted_pref_name); + } + else + { + sql ("INSERT INTO config_preferences" + " (config, type, name, value," + " pref_nvt, pref_id, pref_type, pref_name)" + " VALUES (%llu, 'SERVER_PREFS', '%s', '%s'," + " NULL, NULL, NULL, NULL);", + config, quoted_name, quoted_value); + } g_free (quoted_value); g_free (quoted_name); From 097a8807c126d84128297027691f7d4aebf861a5 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Mon, 11 Mar 2024 11:59:17 +0100 Subject: [PATCH 44/46] Fix: Set pref_... fields in cleanup-config-prefs The --optimize option cleanup-config-prefs now also sets the fields pref_nvt, pref_id, pref_type, pref_name if they are missing or invalid for VT preferences or sets them to NULL for scanner preferences. --- src/manage_sql.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 93284576a..567d8ade6 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -58340,23 +58340,36 @@ manage_optimize (GSList *log_config, const db_conn_info_t *database, } else if (strcasecmp (name, "cleanup-config-prefs") == 0) { - int removed, fixed_values; sql ("DELETE FROM config_preferences WHERE id NOT IN" " (SELECT min(id) FROM config_preferences" " GROUP BY config, type, name, value);"); - removed = sql_changes(); sql ("UPDATE config_preferences" " SET value = (SELECT value FROM nvt_preferences" " WHERE name='scanner_plugins_timeout')" " WHERE name = 'scanner_plugins_timeout'" " AND value = 'SCANNER_NVT_TIMEOUT';"); - fixed_values = sql_changes(); - success_text = g_strdup_printf ("Optimized: cleanup-config-prefs." - " Duplicate config preferences removed:" - " %d. Corrected preference values: %d", - removed, fixed_values); + sql ("UPDATE config_preferences" + " SET pref_nvt = NULL," + " pref_id = NULL," + " pref_type = NULL," + " pref_name = NULL" + " WHERE type = 'SERVER_PREFS' AND pref_nvt IS NOT NULL;"); + + sql ("UPDATE config_preferences" + " SET pref_nvt = substring (name, '^([^:]*)')," + " pref_id = CAST(substring (name, '^[^:]*:([0-9]+)') AS integer)," + " pref_type = substring (name, '^[^:]*:[0-9]+:([^:]*):')," + " pref_name = substring (name, '^[^:]*:[0-9]+:[^:]*:(.*)')" + " WHERE type = 'PLUGINS_PREFS'" + " AND (pref_nvt = '(null)' OR pref_nvt IS NULL" + " OR pref_type = '(null)' OR pref_type IS NULL" + " OR pref_name = '(null)' OR pref_name IS NULL)" + " AND name ~ '^[^:]*:[0-9]+:[^:]*:.*'" + " AND type = 'PLUGINS_PREFS';"); + + success_text = g_strdup_printf ("Optimized: cleanup-config-prefs."); } else if (strcasecmp (name, "cleanup-feed-permissions") == 0) { From f0022dc7569255c9a0a3ba96f3b52edba796477f Mon Sep 17 00:00:00 2001 From: Greenbone Bot Date: Tue, 12 Mar 2024 15:12:53 +0000 Subject: [PATCH 45/46] Automatic release to 23.5.0 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a915ed55a..42839fe83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0) message ("-- Configuring Greenbone Vulnerability Manager...") project (gvm - VERSION 23.4.1 + VERSION 23.5.0 LANGUAGES C) if (POLICY CMP0005) @@ -58,7 +58,7 @@ endif (NOT CMAKE_BUILD_TYPE MATCHES "Release") # Set dev version if this is a development version and not a full release, # unset (put value 0 or delete line) before a full release and reset after. -set (PROJECT_DEV_VERSION 1) +set (PROJECT_DEV_VERSION 0) # If PROJECT_DEV_VERSION is set, the version string will be set to: # "major.minor.patch~dev${PROJECT_DEV_VERSION}${GIT_REVISION}" From 38c6d31904e6a34256484b242d9630a2840093a8 Mon Sep 17 00:00:00 2001 From: Greenbone Bot Date: Tue, 12 Mar 2024 15:12:54 +0000 Subject: [PATCH 46/46] Automatic adjustments after release [skip ci] * Update to version 23.5.1-dev1 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 42839fe83..6f5f64a61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0) message ("-- Configuring Greenbone Vulnerability Manager...") project (gvm - VERSION 23.5.0 + VERSION 23.5.1 LANGUAGES C) if (POLICY CMP0005) @@ -58,7 +58,7 @@ endif (NOT CMAKE_BUILD_TYPE MATCHES "Release") # Set dev version if this is a development version and not a full release, # unset (put value 0 or delete line) before a full release and reset after. -set (PROJECT_DEV_VERSION 0) +set (PROJECT_DEV_VERSION 1) # If PROJECT_DEV_VERSION is set, the version string will be set to: # "major.minor.patch~dev${PROJECT_DEV_VERSION}${GIT_REVISION}"