Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync SCAP using a second schema #1111

Merged
merged 29 commits into from
Jun 3, 2020
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
88eb5de
Remove dummy functions left over from SQLite3 times
mattmundell May 28, 2020
ec4198a
In SCAP sync always update everything
mattmundell May 28, 2020
186e77b
Rebuild using a second SCAP schema
mattmundell May 28, 2020
1efe48e
Correct schema name
mattmundell May 28, 2020
68b45ea
Make the SCAP indexes after the data is inserted
mattmundell May 28, 2020
e6d27a6
Remove flags, they are always updated now
mattmundell May 28, 2020
ac0e21a
Remove updated returns, as everything updates now
mattmundell May 28, 2020
2741bd5
Merge OVAL rebuild in all case
mattmundell May 28, 2020
f02af10
Return empty trigger function
mattmundell May 28, 2020
05b27f1
Move index creation back up, as the data load needs them
mattmundell May 28, 2020
0f23980
Add start of optional CSV loading
mattmundell May 29, 2020
6814e49
Merge branch 'master' into bg-scap-sync
mattmundell Jun 1, 2020
b2cce6a
Fix doc
mattmundell Jun 1, 2020
090a090
Prepend new schema everywhere
mattmundell Jun 1, 2020
e830ca1
Put scap2 first on path, otherwise scap is used
mattmundell Jun 1, 2020
24a147d
Improve names
mattmundell Jun 1, 2020
5187ea3
Use a shared end function
mattmundell Jun 1, 2020
c27b6c7
Remove the gotos, as there's no fail code
mattmundell Jun 1, 2020
2fe4688
Move CSV loading out to function
mattmundell Jun 1, 2020
b23e614
Remove SCAP triggers
mattmundell Jun 1, 2020
9f07794
Remove unused updated status
mattmundell Jun 1, 2020
87c2600
Remove unused OVAL updated status
mattmundell Jun 1, 2020
b926992
Remove old case that was for date check
mattmundell Jun 1, 2020
2a8872f
Remove note, migration is still required
mattmundell Jun 1, 2020
1556859
Remove note, still required
mattmundell Jun 1, 2020
5d8aa40
Update changelog
mattmundell Jun 1, 2020
4f6b17b
Merge branch 'small-scap-cleanups' of mattmundell.github.com:mattmund…
mattmundell Jun 2, 2020
010c8c4
Merge branch 'master' of mattmundell.github.com:greenbone/gvmd into b…
mattmundell Jun 2, 2020
405123c
Merge branch 'master' into bg-scap-sync
timopollmeier Jun 3, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Simplify sync lockfile handling [#1059](https://github.com/greenbone/gvmd/pull/1059)
- Do not ignore empty hosts_allow and ifaces_allow [#1064](https://github.com/greenbone/gvmd/pull/1064)
- Reduce the memory cache of NVTs [#1076](https://github.com/greenbone/gvmd/pull/1076)
- Sync SCAP using a second schema [#1111](https://github.com/greenbone/gvmd/pull/1111)

### Fixed
- Add NULL check in nvts_feed_version_epoch [#768](https://github.com/greenbone/gvmd/pull/768)
Expand Down
186 changes: 69 additions & 117 deletions src/manage_pg.c
Original file line number Diff line number Diff line change
Expand Up @@ -3115,32 +3115,33 @@ manage_db_init (const gchar *name)
}
else if (strcasecmp (name, "scap") == 0)
{
sql ("CREATE OR REPLACE FUNCTION drop_scap () RETURNS void AS $$"
sql ("CREATE OR REPLACE FUNCTION drop_scap2 () RETURNS void AS $$"
" BEGIN"
" IF EXISTS (SELECT schema_name FROM information_schema.schemata"
" WHERE schema_name = 'scap')"
" WHERE schema_name = 'scap2')"
" THEN"
" DROP SCHEMA IF EXISTS scap CASCADE;"
" DROP SCHEMA IF EXISTS scap2 CASCADE;"
" END IF;"
" END;"
" $$ LANGUAGE plpgsql;");

sql ("SELECT drop_scap ();");
sql ("DROP FUNCTION drop_scap ();");
sql ("CREATE SCHEMA scap;");

sql ("SELECT set_config ('search_path',"
" current_setting ('search_path') || ',scap',"
" 'scap2,' || current_setting ('search_path'),"
" false);");

/* Create tables and indexes. */
sql ("SELECT drop_scap2 ();");
sql ("DROP FUNCTION drop_scap2 ();");

sql ("CREATE SCHEMA scap2;");

sql ("CREATE TABLE scap.meta"
/* Create tables. */

sql ("CREATE TABLE scap2.meta"
" (id SERIAL PRIMARY KEY,"
" name text UNIQUE,"
" value text);");

sql ("CREATE TABLE scap.cves"
sql ("CREATE TABLE scap2.cves"
" (id SERIAL PRIMARY KEY,"
" uuid text UNIQUE,"
" name text,"
Expand All @@ -3156,16 +3157,8 @@ manage_db_init (const gchar *name)
" availability_impact text,"
" products text,"
" cvss FLOAT DEFAULT 0);");
sql ("CREATE UNIQUE INDEX cve_idx"
" ON cves (name);");
sql ("CREATE INDEX cves_by_creation_time_idx"
" ON cves (creation_time);");
sql ("CREATE INDEX cves_by_modification_time_idx"
" ON cves (modification_time);");
sql ("CREATE INDEX cves_by_cvss"
" ON cves (cvss);");

sql ("CREATE TABLE scap.cpes"
sql ("CREATE TABLE scap2.cpes"
" (id SERIAL PRIMARY KEY,"
" uuid text UNIQUE,"
" name text,"
Expand All @@ -3178,29 +3171,15 @@ manage_db_init (const gchar *name)
" max_cvss FLOAT DEFAULT 0,"
" cve_refs INTEGER DEFAULT 0,"
" nvd_id text);");
sql ("CREATE UNIQUE INDEX cpe_idx"
" ON cpes (name);");
sql ("CREATE INDEX cpes_by_creation_time_idx"
" ON cpes (creation_time);");
sql ("CREATE INDEX cpes_by_modification_time_idx"
" ON cpes (modification_time);");
sql ("CREATE INDEX cpes_by_cvss"
" ON cpes (max_cvss);");
sql ("CREATE INDEX cpes_by_uuid"
" ON cpes (uuid);");

sql ("CREATE TABLE scap.affected_products"
sql ("CREATE TABLE scap2.affected_products"
" (cve INTEGER NOT NULL,"
" cpe INTEGER NOT NULL,"
" UNIQUE (cve, cpe),"
" FOREIGN KEY(cve) REFERENCES cves(id),"
" FOREIGN KEY(cpe) REFERENCES cpes(id));");
sql ("CREATE INDEX afp_cpe_idx"
" ON affected_products (cpe);");
sql ("CREATE INDEX afp_cve_idx"
" ON affected_products (cve);");

sql ("CREATE TABLE scap.ovaldefs"
sql ("CREATE TABLE scap2.ovaldefs"
" (id SERIAL PRIMARY KEY,"
" uuid text UNIQUE,"
" name text," /* OVAL identifier. */
Expand All @@ -3216,84 +3195,22 @@ manage_db_init (const gchar *name)
" status TEXT,"
" max_cvss FLOAT DEFAULT 0,"
" cve_refs INTEGER DEFAULT 0);");
sql ("CREATE INDEX ovaldefs_idx"
" ON ovaldefs (name);");
sql ("CREATE INDEX ovaldefs_by_creation_time"
" ON ovaldefs (creation_time);");

sql ("CREATE TABLE scap.ovalfiles"
sql ("CREATE TABLE scap2.ovalfiles"
" (id SERIAL PRIMARY KEY,"
" xml_file TEXT UNIQUE);");
sql ("CREATE UNIQUE INDEX ovalfiles_idx"
" ON ovalfiles (xml_file);");

sql ("CREATE TABLE scap.affected_ovaldefs"
sql ("CREATE TABLE scap2.affected_ovaldefs"
" (cve INTEGER NOT NULL,"
" ovaldef INTEGER NOT NULL,"
" FOREIGN KEY(cve) REFERENCES cves(id),"
" FOREIGN KEY(ovaldef) REFERENCES ovaldefs(id));");
sql ("CREATE INDEX aff_ovaldefs_def_idx"
" ON affected_ovaldefs (ovaldef);");
sql ("CREATE INDEX aff_ovaldefs_cve_idx"
" ON affected_ovaldefs (cve);");

/* Create deletion triggers. */

sql ("CREATE OR REPLACE FUNCTION scap_delete_affected ()"
" RETURNS TRIGGER AS $$"
" BEGIN"
" DELETE FROM affected_products where cve = old.id;"
" DELETE FROM affected_ovaldefs where cve = old.id;"
" RETURN old;"
" END;"
"$$ LANGUAGE plpgsql;");

sql ("CREATE TRIGGER cves_delete AFTER DELETE ON cves"
" FOR EACH ROW EXECUTE PROCEDURE scap_delete_affected ();");

sql ("CREATE OR REPLACE FUNCTION scap_update_cpes ()"
" RETURNS TRIGGER AS $$"
" BEGIN"
" UPDATE cpes SET max_cvss = 0.0 WHERE id = old.cpe;"
" UPDATE cpes SET cve_refs = cve_refs -1 WHERE id = old.cpe;"
" RETURN old;"
" END;"
"$$ LANGUAGE plpgsql;");

sql ("CREATE TRIGGER affected_delete AFTER DELETE ON affected_products"
" FOR EACH ROW EXECUTE PROCEDURE scap_update_cpes ();");

sql ("CREATE OR REPLACE FUNCTION scap_delete_oval ()"
" RETURNS TRIGGER AS $$"
" BEGIN"
" DELETE FROM affected_ovaldefs"
" WHERE id IN (SELECT id FROM ovaldefs"
" WHERE ovaldefs.xml_file = old.xml_file);"
" DELETE FROM ovaldefs WHERE ovaldefs.xml_file = old.xml_file;"
" RETURN old;"
" END;"
"$$ LANGUAGE plpgsql;");

sql ("CREATE TRIGGER ovalfiles_delete AFTER DELETE ON ovalfiles"
" FOR EACH ROW EXECUTE PROCEDURE scap_delete_oval ();");

sql ("CREATE OR REPLACE FUNCTION scap_update_oval ()"
" RETURNS TRIGGER AS $$"
" BEGIN"
" UPDATE ovaldefs SET max_cvss = 0.0 WHERE id = old.ovaldef;"
" RETURN old;"
" END;"
"$$ LANGUAGE plpgsql;");

sql ("CREATE TRIGGER affected_ovaldefs_delete"
" AFTER DELETE ON affected_ovaldefs"
" FOR EACH ROW EXECUTE PROCEDURE scap_update_oval ();");

/* Init tables. */

sql ("INSERT INTO scap.meta (name, value)"
sql ("INSERT INTO scap2.meta (name, value)"
" VALUES ('database_version', '16');");
sql ("INSERT INTO scap.meta (name, value)"
sql ("INSERT INTO scap2.meta (name, value)"
" VALUES ('last_update', '0');");
}
else
Expand All @@ -3306,26 +3223,61 @@ manage_db_init (const gchar *name)
}

/**
* @brief Dummy function.
*
* @param[in] name Dummy arg.
*/
void
manage_db_check_mode (const gchar *name)
{
return;
}

/**
* @brief Dummy function.
* @brief Init external database.
*
* @param[in] name Dummy arg.
* @param[in] name Name. Currently only "scap".
*
* @return 0.
* @return 0 success, -1 error.
*/
int
manage_db_check (const gchar *name)
manage_db_init_indexes (const gchar *name)
{
if (strcasecmp (name, "scap") == 0)
{
sql ("CREATE UNIQUE INDEX cve_idx"
" ON scap2.cves (name);");
sql ("CREATE INDEX cves_by_creation_time_idx"
" ON scap2.cves (creation_time);");
sql ("CREATE INDEX cves_by_modification_time_idx"
" ON scap2.cves (modification_time);");
sql ("CREATE INDEX cves_by_cvss"
" ON scap2.cves (cvss);");

sql ("CREATE UNIQUE INDEX cpe_idx"
" ON scap2.cpes (name);");
sql ("CREATE INDEX cpes_by_creation_time_idx"
" ON scap2.cpes (creation_time);");
sql ("CREATE INDEX cpes_by_modification_time_idx"
" ON scap2.cpes (modification_time);");
sql ("CREATE INDEX cpes_by_cvss"
" ON scap2.cpes (max_cvss);");
sql ("CREATE INDEX cpes_by_uuid"
" ON scap2.cpes (uuid);");

sql ("CREATE INDEX afp_cpe_idx"
" ON scap2.affected_products (cpe);");
sql ("CREATE INDEX afp_cve_idx"
" ON scap2.affected_products (cve);");

sql ("CREATE INDEX ovaldefs_idx"
" ON scap2.ovaldefs (name);");
sql ("CREATE INDEX ovaldefs_by_creation_time"
" ON scap2.ovaldefs (creation_time);");

sql ("CREATE UNIQUE INDEX ovalfiles_idx"
" ON scap2.ovalfiles (xml_file);");

sql ("CREATE INDEX aff_ovaldefs_def_idx"
" ON scap2.affected_ovaldefs (ovaldef);");
sql ("CREATE INDEX aff_ovaldefs_cve_idx"
" ON scap2.affected_ovaldefs (cve);");
}
else
{
assert (0);
return -1;
}

return 0;
}

Expand Down
4 changes: 0 additions & 4 deletions src/manage_sql.h
Original file line number Diff line number Diff line change
Expand Up @@ -409,10 +409,6 @@ int manage_cert_db_exists ();

int manage_scap_db_exists ();

void manage_db_check_mode (const gchar *);

int manage_db_check (const gchar *);

int
count (const char *, const get_data_t *, column_t *, column_t *, const char **,
int, const char *, const char *, int);
Expand Down
Loading