Skip to content

Commit

Permalink
Upgrade to cve 5.1 schema (#196)
Browse files Browse the repository at this point in the history
  • Loading branch information
@prabhu
prabhu authored Nov 13, 2024
1 parent 9ae0755 commit 7d1b603
Show file tree
Hide file tree
Showing 20 changed files with 1,942 additions and 296 deletions.
2 changes: 1 addition & 1 deletion INTEGRATION.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ The vulnerability database comprises two SQLite database files.

![Index schema](./docs/vdb-index-schema.png)

- data.vdb6 - Full CVE source database containing normalized data in CVE 5.0 specification formation and purl prefix.
- data.vdb6 - Full CVE source database containing normalized data in CVE 5.1 specification formation and purl prefix.

![Data schema](./docs/vdb-schema.png)

Expand Down
10 changes: 5 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ A good vulnerability database must have the following properties:
- Easy to [download](#download-pre-built-database-recommended), [integrate](./INTEGRATION.md), and use
- Performance

Multiple upstream sources are used by vdb to improve accuracy and reduce false negatives. SQLite database containing data in CVE 5.0 schema format is precompiled and distributed as files via ghcr to simplify download. With automatic purl prefix generation even for git repos, searches on the database can be performed with purl, cpe, or even http git url string. Every row in the database uses an open specification such as CVE 5.0 or Package URL (purl and vers) thus preventing the possibility of vendor lock-in.
Multiple upstream sources are used by vdb to improve accuracy and reduce false negatives. SQLite database containing data in CVE 5.1 schema format is precompiled and distributed as files via ghcr to simplify download. With automatic purl prefix generation even for git repos, searches on the database can be performed with purl, cpe, or even http git url string. Every row in the database uses an open specification such as CVE 5.0 or Package URL (purl and vers) thus preventing the possibility of vendor lock-in.

## Vulnerability Data sources

Expand Down Expand Up @@ -40,7 +40,7 @@ Multiple upstream sources are used by vdb to improve accuracy and reduce false n
## Installation

```shell
pip install appthreat-vulnerability-db>=6.0.1
pip install appthreat-vulnerability-db>=6.2.0
```

To install vdb with optional dependencies such as `oras` use the `[oras]` or `[all]` dependency group.
Expand All @@ -49,10 +49,10 @@ To install vdb with optional dependencies such as `oras` use the `[oras]` or `[a
pip install appthreat-vulnerability-db[all]
```

**NOTE:** VDB v6 is a major rewrite to use SQLite database. Current users of depscan v5 must continue using version 5.6.x
**NOTE:** VDB v6 is a major rewrite to use SQLite database. Current users of depscan v5 must continue using version 5.8.x

```shell
pip install appthreat-vulnerability-db==5.6.7
pip install appthreat-vulnerability-db==5.8.0
```

## Usage
Expand Down Expand Up @@ -93,7 +93,7 @@ Use any sqlite browser or cli tools to load and query the two databases.

<img src="./docs/index-vdb6.png" alt="index" width="400">

**data.vdb6** - Contains source data in CVE 5.0 format stored as a jsonb blob.
**data.vdb6** - Contains source data in CVE 5.1 format stored as a jsonb blob.

<img src="./docs/vdb6.png" alt="database" width="400">

Expand Down
93 changes: 62 additions & 31 deletions data/CVE_JSON_5.0_schema.json → data/CVE_Record_Format.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@
]
}
}
}
},
"additionalProperties": false
},
"cveId": {
"type": "string",
Expand Down Expand Up @@ -74,8 +75,7 @@
},
"timestamp": {
"type": "string",
"format": "date-time",
"description": "Date/time format based on RFC3339 and ISO ISO8601, with an optional timezone in the format 'yyyy-MM-ddTHH:mm:ssZZZZ'. If timezone offset is not given, GMT (0000) is assumed.",
"description": "Date/time format based on RFC3339 and ISO ISO8601, with an optional timezone in the format 'yyyy-MM-ddTHH:mm:ss[+-]ZH:ZM'. If timezone offset is not given, GMT (+00:00) is assumed.",
"pattern": "^(((2000|2400|2800|(19|2[0-9](0[48]|[2468][048]|[13579][26])))-02-29)|(((19|2[0-9])[0-9]{2})-02-(0[1-9]|1[0-9]|2[0-8]))|(((19|2[0-9])[0-9]{2})-(0[13578]|10|12)-(0[1-9]|[12][0-9]|3[01]))|(((19|2[0-9])[0-9]{2})-(0[469]|11)-(0[1-9]|[12][0-9]|30)))T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})?$"
},
"version": {
Expand Down Expand Up @@ -246,7 +246,8 @@
"minLength": 1,
"maxLength": 4096
}
}
},
"additionalProperties": false
}
},
"platforms": {
Expand Down Expand Up @@ -284,10 +285,13 @@
},
{
"required": ["version", "status", "versionType"],
"oneOf": [
{"required": ["lessThan"]},
{"required": ["lessThanOrEqual"]}
]
"maxProperties": 3
},
{
"required": ["version", "status", "versionType", "lessThan"]
},
{
"required": ["version", "status", "versionType", "lessThanOrEqual"]
}
],
"properties": {
Expand Down Expand Up @@ -330,6 +334,7 @@
"type": "object",
"description": "The start of a single status change during the range.",
"required": ["at", "status"],
"additionalProperties": false,
"properties": {
"at": {
"description": "The version at which a status change occurs.",
Expand All @@ -342,7 +347,8 @@
}
}
}
}
},
"additionalProperties": false
}
}
}
Expand All @@ -355,9 +361,10 @@
]
},
"dataVersion": {
"description": "The version of the schema being used. Used to support multiple versions of this format.",
"description": "The version of the CVE schema used for validating this record. Used to support multiple versions of this format.",
"type": "string",
"enum": ["5.0"]
"pattern": "^5\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?$",
"default": "5.1.0"
},
"cveMetadataPublished": {
"description": "This is meta data about the CVE ID such as the CVE ID, who requested it, who assigned it, when it was requested, the current state (PUBLISHED, REJECTED, etc.) and so on. These fields are controlled by the CVE Services.",
Expand Down Expand Up @@ -477,7 +484,8 @@
"description": "Timestamp to be set by the system of record at time of submission. If dateUpdated is provided to the system of record it will be replaced by the current timestamp at the time of submission."
}
},
"required": ["orgId"]
"required": ["orgId"],
"additionalProperties": false
},
"cnaPublishedContainer": {
"description": "An object containing the vulnerability information provided by a CVE Numbering Authority (CNA) for a published CVE ID. There can only be one CNA container per CVE record since there can only be one assigning CNA. The CNA container must include the required information defined in the CVE Rules, which includes a product, version, problem type, prose description, and a reference.",
Expand Down Expand Up @@ -555,6 +563,7 @@
"patternProperties": {
"^x_[^.]*$": {}
},
"$comment": "The character . is restricted in names allowed by patternProperties to work-around naming limitations in some common implementations.",
"additionalProperties": false
},
"cnaRejectedContainer": {
Expand Down Expand Up @@ -585,6 +594,7 @@
"patternProperties": {
"^x_[^.]*$": {}
},
"$comment": "The character . is restricted in names allowed by patternProperties to work-around naming limitations in some common implementations.",
"additionalProperties": false
},
"adpContainer": {
Expand Down Expand Up @@ -657,6 +667,7 @@
"patternProperties": {
"^x_[^.]*$": {}
},
"$comment": "The character . is restricted in names allowed by patternProperties to work-around naming limitations in some common implementations.",
"additionalProperties": false
},
"affected": {
Expand Down Expand Up @@ -715,7 +726,8 @@
"required": [
"type",
"value"
]
],
"additionalProperties": false
}
}
},
Expand All @@ -729,7 +741,8 @@
"type": "object",
"description": "A description with lang set to an English language (en, en_US, en_UK, and so on).",
"properties": {"lang": {"$ref": "#/definitions/englishLanguage"}},
"required": ["lang"]
"required": ["lang"],
"$comment": "Cannot use additionalProperties: false here, as this prevents the other properties used by /definitions/description."
},
"descriptions": {
"type": "array",
Expand Down Expand Up @@ -780,12 +793,14 @@
"maxLength": 128
},
"references": {"$ref": "#/definitions/references"}
}
},
"additionalProperties": false
},
"minItems": 1,
"uniqueItems": true
}
}
},
"additionalProperties": false
},
"minItems": 1,
"uniqueItems": true
Expand Down Expand Up @@ -819,7 +834,8 @@
"description": "Prose description of the impact scenario. At a minimum provide the description given by CAPEC.",
"$ref": "#/definitions/descriptions"
}
}
},
"additionalProperties": false
}
},
"metrics": {
Expand All @@ -829,8 +845,11 @@
"uniqueItems": true,
"items": {
"type": "object",
"description": "This is impact type information (e.g. a text description, CVSSv2, CVSSv3, etc.). Must contain: At least one entry, can be text, CVSSv2, CVSSv3, others may be added.",
"description": "This is impact type information (e.g. a text description, CVSSv2, CVSSv3, CVSSV4, etc.). Must contain: At least one entry, can be text, CVSSv2, CVSSv3, others may be added.",
"anyOf": [
{
"required": ["cvssV4_0"]
},
{
"required": ["cvssV3_1"]
},
Expand Down Expand Up @@ -871,9 +890,11 @@
"required": [
"lang",
"value"
]
],
"additionalProperties": false
}
},
"cvssV4_0": {"$ref": "imports/cvss/cvss-v4.0.json"},
"cvssV3_1": {"$ref": "imports/cvss/cvss-v3.1.json"},
"cvssV3_0": {"$ref": "imports/cvss/cvss-v3.0.json"},
"cvssV2_0": {"$ref": "imports/cvss/cvss-v2.0.json"},
Expand All @@ -893,12 +914,15 @@
},
"content": {
"type": "object",
"$comment": "additionalProperties are allowed here, since this construct supports arbitrary JSON.",
"description": "JSON object not covered by another metrics format.",
"minProperties": 1
}
}
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}
},
"configurations": {
Expand Down Expand Up @@ -951,7 +975,7 @@
],
"properties": {
"time": {
"description": "Timestamp representing when the event in the timeline occurred. The timestamp format is based on RFC3339 and ISO ISO8601, with an optional timezone. yyyy-MM-ddTHH:mm:ssZZZZ - if the timezone offset is not given, GMT (0000) is assumed.",
"description": "Timestamp representing when the event in the timeline occurred. The timestamp format is based on RFC3339 and ISO ISO8601, with an optional timezone. yyyy-MM-ddTHH:mm:ss[+-]ZH:ZM - if the timezone offset is not given, GMT (+00:00) is assumed.",
"$ref": "#/definitions/timestamp"
},
"lang": {
Expand All @@ -964,7 +988,8 @@
"minLength": 1,
"maxLength": 4096
}
}
},
"additionalProperties": false
}
},
"credits": {
Expand Down Expand Up @@ -1006,6 +1031,7 @@
]
}
},
"additionalProperties": false,
"required": [
"lang",
"value"
Expand Down Expand Up @@ -1035,15 +1061,15 @@
"uniqueItems": true,
"items": {
"type": "object",
"description": "",
"description": "A taxonomy mapping object identifies the taxonomy by a name and version (eg., ATT&CK v13.1, CVSS 3.1, CWE 4.12) along with a list of relations relevant to this CVE.",
"required": [
"taxonomyName",
"taxonomyRelations"
],
"properties": {
"taxonomyName": {
"type": "string",
"description": "The name of the taxonomy.",
"description": "The name of the taxonomy, eg., ATT&CK, D3FEND, CWE, CVSS",
"minLength": 1,
"maxLength": 128
},
Expand All @@ -1055,12 +1081,12 @@
},
"taxonomyRelations": {
"type": "array",
"description": "",
"description": "List of relationships to the taxonomy for the vulnerability.",
"minItems": 1,
"uniqueItems": true,
"items": {
"type": "object",
"description": "List of relationships to the taxonomy for the vulnerability. Relationships can be between the taxonomy and the CVE or two taxonomy items.",
"description": "A relationship between the taxonomy and the CVE or two taxonomy items.",
"required": [
"taxonomyId",
"relationshipName",
Expand All @@ -1085,17 +1111,20 @@
"minLength": 1,
"maxLength": 2048
}
}
},
"additionalProperties": false
}
}
}
},
"additionalProperties": false
}
},
"tagExtension": {
"type": "string",
"minLength": 2,
"maxLength": 128,
"pattern": "^x_.*$"
"pattern": "^x_.*$",
"$comment": "These values are not used as JSON property names, so there is not a need to work-around property naming limitations in some common implementations."
},
"cnaTags": {
"type": "array",
Expand Down Expand Up @@ -1134,6 +1163,7 @@
{
"title": "Published",
"description": "When a CNA populates the data associated with a CVE ID as a CVE Record, the state of the CVE Record is Published.",
"type": "object",
"properties": {
"dataType": {
"$ref": "#/definitions/dataType"
Expand Down Expand Up @@ -1171,6 +1201,7 @@
{
"title": "Rejected",
"description": "If the CVE ID and associated CVE Record should no longer be used, the CVE Record is placed in the Rejected state. A Rejected CVE Record remains on the CVE List so that users can know when it is invalid.",
"type": "object",
"properties": {
"dataType": {
"$ref": "#/definitions/dataType"
Expand Down Expand Up @@ -1200,4 +1231,4 @@
"additionalProperties": false
}
]
}
}
2 changes: 1 addition & 1 deletion data/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ Modify the jsonschema to make it compatible with datamodel-code-generator which

```shell
pip install datamodel-code-generator
datamodel-codegen --input data/CVE_JSON_5.0_schema.json --input-file-type jsonschema --output vdb/lib/cve_model --output-model-type pydantic_v2.BaseModel --target-python-version 3.10 --use-annotated --class-name CVE
datamodel-codegen --input data/CVE_Record_Format.json --input-file-type jsonschema --output vdb/lib/cve_model --output-model-type pydantic_v2.BaseModel --target-python-version 3.10 --use-annotated --class-name CVE
```
Loading

0 comments on commit 7d1b603

Please sign in to comment.