Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up regular result iterator #1370

Merged
merged 22 commits into from
Dec 8, 2020
Merged

Speed up regular result iterator #1370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
008db6c
Correct count in check for all levels
mattmundell Dec 1, 2020
edf3c69
Use nvts table for cvss_base in PRE_BASE_RESULT_ITERATOR_COLUMNS
mattmundell Dec 1, 2020
ac07f1b
Get new severity with LATERAL in plain result iterator
mattmundell Dec 1, 2020
cee597b
Neaten if blocks
mattmundell Dec 1, 2020
185b6f4
Use standard identifier style
mattmundell Dec 1, 2020
fd864df
Correct dynamic case
mattmundell Dec 1, 2020
6a937e7
Wrap results.severity in SELECT
mattmundell Dec 1, 2020
401dec8
Merge branch 'master' into lateral-results
mattmundell Dec 1, 2020
54a5c4c
Also use LATERAL for the dynamic no overrides case
mattmundell Dec 1, 2020
7431afe
Inline current_severity because we have joined table nvts
mattmundell Dec 1, 2020
75cfcd0
Also use lateral for the no dynamic no overrides case
mattmundell Dec 1, 2020
1158377
Use more specific new severity views for result iterator
mattmundell Dec 1, 2020
8c87020
Remove redundant _D and _OD defs
mattmundell Dec 1, 2020
d616fa2
Inline the result_new_severities_dynamic view to get access to joined…
mattmundell Dec 2, 2020
91f4e35
Move current severity into WITH in the overrides dynamic case
mattmundell Dec 2, 2020
63f50e0
Add doc
mattmundell Dec 2, 2020
26966f6
Move def out of function
mattmundell Dec 2, 2020
261f906
Update changelog
mattmundell Dec 2, 2020
d746baa
Add LATERAL to result_count as column may be used by filtering
mattmundell Dec 2, 2020
47c1bfa
Move common LATERAL code to function
mattmundell Dec 2, 2020
938384d
Merge branch 'master' into lateral-results
mattmundell Dec 2, 2020
5876d4e
Add LATERAL to type_build_select for result aggregates
mattmundell Dec 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Add `allow_simult_ips_same_host` field for targets [#1346](https://github.com/greenbone/gvmd/pull/1346)
- Speed up GET_VULNS [#1354](https://github.com/greenbone/gvmd/pull/1354) [#1355](https://github.com/greenbone/gvmd/pull/1354)
- Speed up result counting iterator [#1358](https://github.com/greenbone/gvmd/pull/1358) [#1361](https://github.com/greenbone/gvmd/pull/1361)
- Speed up result iterator [#1370](https://github.com/greenbone/gvmd/pull/1358) [#1361](https://github.com/greenbone/gvmd/pull/1370)
- Improve GMP docs around users [#1363](https://github.com/greenbone/gvmd/pull/1363)

### Changed
Expand Down
217 changes: 92 additions & 125 deletions src/manage_sql.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5644,6 +5644,8 @@ count2 (const char *type, const get_data_t *get, column_t *select_columns,
else
filter = NULL;

g_debug ("%s", __func__);

clause = filter_clause (type, filter ? filter : get->filter, filter_columns,
get->trash && trash_select_columns
? trash_select_columns
Expand Down Expand Up @@ -5712,6 +5714,9 @@ count2 (const char *type, const get_data_t *get, column_t *select_columns,
g_free (columns);
g_free (owned_clause);
g_free (clause);

g_debug ("%s: done", __func__);

return ret;
}

Expand Down Expand Up @@ -21213,7 +21218,7 @@ where_levels_auto (const char *levels, const char *new_severity_sql)

g_string_append (levels_sql, ")");

if (count == 6)
if (count == 5)
{
/* All levels. */
g_string_free (levels_sql, TRUE);
Expand Down Expand Up @@ -21426,18 +21431,20 @@ where_qod (int min_qod)
/* ^ 10 = 0 */ \
{ "port", "location", KEYWORD_TYPE_STRING }, \
{ "nvt", NULL, KEYWORD_TYPE_STRING }, \
{ "severity_to_type (severity)", "original_type", KEYWORD_TYPE_STRING }, \
{ "severity_to_type (results.severity)", \
"original_type", \
KEYWORD_TYPE_STRING }, \
{ "severity_to_type (" new_severity_sql ")", \
"type", \
KEYWORD_TYPE_STRING }, \
{ "description", NULL, KEYWORD_TYPE_STRING }, \
{ "task", NULL, KEYWORD_TYPE_INTEGER }, \
{ "report", "report_rowid", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT cvss_base FROM nvts WHERE nvts.oid = nvt)", \
{ "nvts.cvss_base", \
"cvss_base", \
KEYWORD_TYPE_DOUBLE }, \
{ "nvt_version", NULL, KEYWORD_TYPE_STRING }, \
{ "severity", "original_severity", KEYWORD_TYPE_DOUBLE }, \
{ "results.severity", "original_severity", KEYWORD_TYPE_DOUBLE }, \
/* ^ 20 = 10 */ \
{ new_severity_sql, \
"severity", \
Expand Down Expand Up @@ -21553,25 +21560,7 @@ where_qod (int min_qod)
* @brief Result iterator columns.
*/
#define BASE_RESULT_ITERATOR_COLUMNS \
PRE_BASE_RESULT_ITERATOR_COLUMNS("results.severity")

/**
* @brief Result iterator columns.
*/
#define BASE_RESULT_ITERATOR_COLUMNS_D \
PRE_BASE_RESULT_ITERATOR_COLUMNS("current_severity (results.severity," \
" results.nvt)")

/**
* @brief Result iterator columns.
*/
#define BASE_RESULT_ITERATOR_COLUMNS_OD \
PRE_BASE_RESULT_ITERATOR_COLUMNS("(SELECT new_severity" \
" FROM result_new_severities" \
" WHERE result_new_severities.result = results.id" \
" AND result_new_severities.user = opts.user_id" \
" AND result_new_severities.dynamic = opts.dynamic" \
" LIMIT 1)")
PRE_BASE_RESULT_ITERATOR_COLUMNS("lateral_new_severity.new_severity")

/**
* @brief Result iterator columns.
Expand All @@ -21588,36 +21577,6 @@ where_qod (int min_qod)
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Result iterator columns.
*/
#define RESULT_ITERATOR_COLUMNS_D \
{ \
BASE_RESULT_ITERATOR_COLUMNS_D \
{ SECINFO_SQL_RESULT_CERT_BUNDS, \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ SECINFO_SQL_RESULT_DFN_CERTS, \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Result iterator columns.
*/
#define RESULT_ITERATOR_COLUMNS_OD \
{ \
BASE_RESULT_ITERATOR_COLUMNS_OD \
{ SECINFO_SQL_RESULT_CERT_BUNDS, \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ SECINFO_SQL_RESULT_DFN_CERTS, \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Result iterator columns, when CERT db is not loaded.
*/
Expand All @@ -21633,36 +21592,6 @@ where_qod (int min_qod)
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Result iterator columns, when CERT db is not loaded.
*/
#define RESULT_ITERATOR_COLUMNS_D_NO_CERT \
{ \
BASE_RESULT_ITERATOR_COLUMNS_D \
{ "0", \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ "0", \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Result iterator columns, when CERT db is not loaded.
*/
#define RESULT_ITERATOR_COLUMNS_OD_NO_CERT \
{ \
BASE_RESULT_ITERATOR_COLUMNS_OD \
{ "0", \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ "0", \
NULL, \
KEYWORD_TYPE_INTEGER }, \
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Generate the extra_tables string for a result iterator.
*
Expand Down Expand Up @@ -21999,7 +21928,7 @@ init_result_get_iterator_severity (iterator_t* iterator, const get_data_t *get,
= "coalesce (results.severity, results.severity)";
}

columns[0].select = "lateralSeverity";
columns[0].select = "lateral_severity";
columns[0].filter = "severity";
columns[0].type = KEYWORD_TYPE_DOUBLE;

Expand All @@ -22009,20 +21938,20 @@ init_result_get_iterator_severity (iterator_t* iterator, const get_data_t *get,

opts = result_iterator_opts_table (apply_overrides,
dynamic_severity);
extra_tables = g_strdup_printf (", LATERAL %s AS lateralSeverity%s",
extra_tables = g_strdup_printf (", LATERAL %s AS lateral_severity%s",
lateral, opts);
g_free (opts);

extra_where = results_extra_where (get->trash, report, host,
apply_overrides, dynamic_severity,
filter ? filter : get->filter,
"lateralSeverity");
"lateral_severity");

extra_where_single = results_extra_where (get->trash, report, host,
apply_overrides,
dynamic_severity,
"min_qod=0",
"lateralSeverity");
"lateral_severity");

free (filter);

Expand Down Expand Up @@ -22092,6 +22021,55 @@ init_result_get_iterator_severity (iterator_t* iterator, const get_data_t *get,
return ret;
}

/**
* @brief SQL for getting current severity.
*/
#define CURRENT_SEVERITY_SQL \
"coalesce ((CASE WHEN results.severity > " G_STRINGIFY (SEVERITY_LOG) \
" THEN CAST (nvts.cvss_base AS double precision)" \
" ELSE results.severity" \
" END)," \
" results.severity)"

/**
* @brief Get LATERAL clause for result iterator.
*
* @param[in] apply_overrides Whether to apply overrides.
* @param[in] dynamic_severity Whether to use dynamic severity.
*
* @return SQL clause for FROM.
*/
static const gchar *
result_iterator_lateral (int apply_overrides, int dynamic_severity)
{
if (apply_overrides && dynamic_severity)
/* Overrides, dynamic. */
return "(WITH curr AS (SELECT " CURRENT_SEVERITY_SQL " AS curr_severity)"
" SELECT coalesce ((SELECT ov_new_severity FROM result_overrides"
" WHERE result = results.id"
" AND result_overrides.user = opts.user_id"
" AND severity_matches_ov"
" ((SELECT curr_severity FROM curr LIMIT 1),"
" ov_old_severity)"
" LIMIT 1),"
" (SELECT curr_severity FROM curr LIMIT 1))"
" AS new_severity)";
if (apply_overrides)
/* Overrides, no dynamic. */
return "(SELECT new_severity"
" FROM result_new_severities_static"
" WHERE result_new_severities_static.result = results.id"
" AND result_new_severities_static.user = opts.user_id"
" LIMIT 1)";
if (dynamic_severity)
/* No overrides, dynamic. */
return "(SELECT " CURRENT_SEVERITY_SQL " AS new_severity)";
/* No overrides, no dynamic.
*
* SELECT because results.severity gives syntax error. */
return "(SELECT results.severity AS new_severity)";
}

/**
* @brief Initialise a result iterator.
*
Expand All @@ -22111,16 +22089,14 @@ init_result_get_iterator (iterator_t* iterator, const get_data_t *get,
{
static const char *filter_columns[] = RESULT_ITERATOR_FILTER_COLUMNS;
static column_t columns[] = RESULT_ITERATOR_COLUMNS;
static column_t columns_dynamic[] = RESULT_ITERATOR_COLUMNS_D;
static column_t columns_overrides_dynamic[] = RESULT_ITERATOR_COLUMNS_OD;
static column_t columns_no_cert[] = RESULT_ITERATOR_COLUMNS_NO_CERT;
static column_t columns_dynamic_no_cert[] = RESULT_ITERATOR_COLUMNS_D_NO_CERT;
static column_t columns_overrides_dynamic_no_cert[] = RESULT_ITERATOR_COLUMNS_OD_NO_CERT;
int ret;
gchar *filter, *extra_tables, *extra_where, *extra_where_single, *opts_tables;
int apply_overrides, dynamic_severity;
column_t *actual_columns;

g_debug ("%s", __func__);

if (report == -1)
{
init_iterator (iterator, "SELECT NULL WHERE false;");
Expand All @@ -22141,40 +22117,17 @@ init_result_get_iterator (iterator_t* iterator, const get_data_t *get,
dynamic_severity = setting_dynamic_severity_int ();

if (manage_cert_loaded ())
{
if (apply_overrides)
/* Overrides, maybe dynamic. */
actual_columns = columns_overrides_dynamic;
else
{
if (dynamic_severity)
/* Dynamic, no overrides. */
actual_columns = columns_dynamic;
else
/* No dynamic, no overrides. */
actual_columns = columns;
}
}
actual_columns = columns;
else
{
if (apply_overrides)
/* Overrides, maybe dynamic. */
actual_columns = columns_overrides_dynamic_no_cert;
else
{
if (dynamic_severity)
/* Dynamic, no overrides. */
actual_columns = columns_dynamic_no_cert;
else
/* No dynamic, no overrides. */
actual_columns = columns_no_cert;
}
}
actual_columns = columns_no_cert;

opts_tables = result_iterator_opts_table (apply_overrides, dynamic_severity);
extra_tables = g_strdup_printf (" LEFT OUTER JOIN nvts"
" ON results.nvt = nvts.oid %s",
opts_tables);
" ON results.nvt = nvts.oid %s,"
" LATERAL %s AS lateral_new_severity",
opts_tables,
result_iterator_lateral (apply_overrides,
dynamic_severity));
g_free (opts_tables);

extra_where = results_extra_where (get->trash, report, host,
Expand Down Expand Up @@ -22210,6 +22163,9 @@ init_result_get_iterator (iterator_t* iterator, const get_data_t *get,
g_free (extra_tables);
g_free (extra_where);
g_free (extra_where_single);

g_debug ("%s: done", __func__);

return ret;
}

Expand Down Expand Up @@ -22250,8 +22206,11 @@ result_count (const get_data_t *get, report_t report, const char* host)

opts_tables = result_iterator_opts_table (apply_overrides, dynamic_severity);
extra_tables = g_strdup_printf (" LEFT OUTER JOIN nvts"
" ON results.nvt = nvts.oid %s",
opts_tables);
" ON results.nvt = nvts.oid %s,"
" LATERAL %s AS lateral_new_severity",
opts_tables,
result_iterator_lateral (apply_overrides,
dynamic_severity));
g_free (opts_tables);

extra_where = results_extra_where (get->trash, report, host,
Expand Down Expand Up @@ -54756,11 +54715,19 @@ type_build_select (const char *type, const char *columns_str,
if (strcasecmp (type, "RESULT") == 0)
{
gchar *original;
int overrides, dynamic;

overrides = filter_term_apply_overrides (filter ? filter : get->filter);
dynamic = setting_dynamic_severity_int ();

original = opts_table;

opts_table = g_strdup_printf (" LEFT OUTER JOIN nvts"
" ON results.nvt = nvts.oid %s",
original);
" ON results.nvt = nvts.oid %s,"
" LATERAL %s AS lateral_new_severity",
original,
result_iterator_lateral (overrides,
dynamic));
g_free (original);
}

Expand Down