diff --git a/bin/dump.py b/bin/dump.py index c1151674..4f75dba8 100755 --- a/bin/dump.py +++ b/bin/dump.py @@ -11,6 +11,9 @@ from vulnerabilitylookup import VulnerabilityLookup from vulnerabilitylookup.default import get_config, get_homedir, safe_create_dir +from website.models import Bundle +from website.models import Comment +from website.web.bootstrap import application logging.config.dictConfig(get_config('logging')) @@ -31,19 +34,41 @@ def dump(self, feed: str, /) -> None: with dest_file.open('a') as f: json.dump(vuln, f) + def dump_comments(self) -> None: + dest_file = self.root_dumps / 'comments.ndjson' + dest_file.unlink(missing_ok=True) + with application.app_context(): + for comment in Comment.query.filter(): + with dest_file.open('a') as f: + json.dump(comment.to_dict(), f) + + def dump_bundles(self) -> None: + dest_file = self.root_dumps / 'bundles.ndjson' + dest_file.unlink(missing_ok=True) + with application.app_context(): + for bundle in Bundle.query.filter(): + with dest_file.open('a') as f: + json.dump(bundle.to_dict(), f) + def main() -> None: parser = argparse.ArgumentParser( description="Dump vulnerability-lookup storage in NDJSON." ) parser.add_argument( - "--feed", help="Feed to dump. Default is set to nvd.", default="nvd" + "--feed", help="Feed to dump. Default is set to nvd.", default="csaf_ox" ) parser.add_argument( "--all", help="Dump all feeds. Default is false", default=False, action="store_true" ) parser.add_argument( - "--index", help="Generate an index to publish all the feeds. Default is false", default=False, action="store_true" + "--index", help="Generate an index to publish all the feeds. Default is false.", default=False, action="store_true" + ) + parser.add_argument( + "--comments", help="Export the comments. Default is false.", action="store_true" + ) + parser.add_argument( + "--bundles", help="Export the bundles. Default is false.", action="store_true" ) args = parser.parse_args() d = Dump() @@ -68,6 +93,12 @@ def main() -> None: index = index + f'
Generated on {current_date}' with dest_file.open('w') as f: f.write(index) + if args.comments: + print('Dumping comments...') + d.dump_comments() + if args.bundles: + print('Dumping bundles...') + d.dump_bundles() if __name__ == '__main__': main() diff --git a/website/models/bundle.py b/website/models/bundle.py index 607da440..1ffd35da 100644 --- a/website/models/bundle.py +++ b/website/models/bundle.py @@ -1,5 +1,6 @@ import json import uuid +from typing import Any from sqlalchemy import func from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import validates @@ -53,3 +54,6 @@ def as_json(self) -> str: "meta": self.meta, } ) + + def to_dict(self) -> dict[str, Any]: + return json.loads(self.as_json()) diff --git a/website/models/comment.py b/website/models/comment.py index 3f94a228..c04d3fda 100644 --- a/website/models/comment.py +++ b/website/models/comment.py @@ -1,4 +1,6 @@ +import json import uuid +from typing import Any from sqlalchemy import func from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import validates @@ -38,3 +40,23 @@ def validates_description_format(self, key: str, value: str) -> str: "value must be 'markdown' or 'text'." ) return value.lower() + + def as_json(self) -> str: + return json.dumps( + { + "uuid": str(self.uuid), + "vulnerability_lookup_origin": str(self.vulnerability_lookup_origin), + "title": self.title, + "description": self.description, + "description_format": self.description_format, + "creation_timestamp": self.creation_timestamp.strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ), + "timestamp": self.timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "related_vulnerabilities": self.related_vulnerabilities, + "meta": self.meta, + } + ) + + def to_dict(self) -> dict[str, Any]: + return json.loads(self.as_json())