Fix CVE feed: comply with the JSON feed specifications and add the full JSON feed object in the script output to add last_updated root fields

sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh

@@ -17,17 +17,18 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
-#install python-pip3
+# install python-pip3
 apt-get update
 apt-get install -y python3-pip
 
-#install requests module
+# install requests module
 pip3 install requests
 
-#python script to generate official-cve-feed.json 
-python3 fetch-official-cve-feed.py
+# python script to generate official-cve-feed.json 
+# tee duplicates the output from the script to stdout for logs and the JSON file
+python3 fetch-official-cve-feed.py | tee official_cve_feed.json
 
-#function to calculate the hash value of official-cve-feed.json 
+# function to calculate the hash value of official-cve-feed.json 
 calculate_hash(){
     if command -v shasum >/dev/null 2>&1; then
     cat "$@" | shasum -a 256 | cut -d' ' -f1
@@ -39,12 +40,13 @@ calculate_hash(){
     fi
 }
 
-#check if official-cve-feed.json blob exists in the bucket
+# check if official-cve-feed.json blob exists in the bucket
 set -e
 EXIT_CODE=0
 gsutil ls gs://k8s-cve-feed/official-cve-feed.json >/dev/null 2>&1 || EXIT_CODE=$?
 
-#fetch the hash value of existing official-cve-feed.json json, if differs then upload the new cve feed data to the existing blob.
+# fetch the hash value of existing official-cve-feed.json json, if differs then
+# upload the new cve feed data to the existing blob.
 if [[ $EXIT_CODE -eq 1  ]]; then 
     gsutil cp official-cve-feed.json gs://k8s-cve-feed
     calculate_hash official-cve-feed.json > cve-feed-hash

sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py

@@ -16,6 +16,7 @@
 
 import json
 import requests
+from datetime import datetime
 
 url = 'https://api.github.com/search/issues?q=is:issue+label:official-cve-feed+\
 state:closed+repo:kubernetes/kubernetes&per_page=100'
@@ -33,34 +34,55 @@
     res = requests.get(res.links['next']['url'], headers=headers)
     gh_items.extend(res.json()['items'])
 
-cve_list = []
+feed_envelope = {
+    'version': 'https://jsonfeed.org/version/1.1',
+    'title': 'Auto-refreshing Official CVE Feed',
+    'home_page_url': 'https://kubernetes.io',
+    'feed_url': 'https://kubernetes.io/docs/reference/issues-security/official-cve-feed/index.json',
+    'description': 'Auto-refreshing official CVE feed for Kubernetes repository',
+    'authors': [
+        {
+            'name': 'Kubernetes Community',
+            'url': 'https://www.kubernetes.dev'
+        }
+    ],
+    '_kubernetes_io': None,
+    'items': None,
+}
+# format the timestamp the same way as GitHub RFC 3339 timestamps, with only seconds and not milli and microseconds.
+root_kubernetes_io = {'updated_at': datetime.utcnow().isoformat(sep='T', timespec='seconds') + 'Z'}
+feed_envelope['_kubernetes_io'] = root_kubernetes_io
 
+cve_list = []
 for item in gh_items:
-    cve = {"issue_url": None, "number": None, "cve_id": None,
-           "summary": None, "cve_url": None, "google_group_url": None}
-    cve['issue_url'] = item['html_url']
-    cve['number'] = item['number']
-    title = item['title'].replace(" -", ":")
-    title = title.split(": ")
-    if len(title) == 1:
-        cve_id = None
-        cve['cve_id'] = None
-        cve['cve_url'] = None
-        cve['summary'] = title[0]
-        cve['google_group_url'] = None
-    else:
-        cve_id = title[0]
-        cve['cve_id'] = title[0]
-        if len(title) == 3:
-            cve['summary'] = title[2]
-        else:
-            cve['summary'] = title[1]
-        cve['cve_url'] = f"https://www.cve.org/cverecord?id={cve_id}"
-        cve['google_group_url'] = \
-        f"https://groups.google.com/g/kubernetes-announce/search?q={cve_id}"
+    # These keys respects the item jsonfeed spec https://www.jsonfeed.org/version/1.1/
+    cve = {'content_text': None, 'date_published': None, 'external_url': None,
+    'id': None,'summary': None, 'url': None, '_kubernetes_io': None}
+    # This is a custom extension
+    item_kubernetes_io = {'google_group_url': None, 'issue_number': None}
+    cve['_kubernetes_io'] = item_kubernetes_io
+
+    cve['url'] = item['html_url']
+    cve['_kubernetes_io']['issue_number'] = item['number']
+    cve['content_text'] = item['body']
+    cve['date_published'] = item['created_at']
+    # This is because some CVEs were titled "CVE-XXXX-XXXX - Something" instead of
+    # "CVE-XXXX-XXXX: Something" on GitHub (see https://github.com/kubernetes/kubernetes/issues/60813).
+    title = item['title'].replace(' -', ':')
+    # This splits the CVE into its ID and the description/name, however some are in the following forms:
+    # - CVE-2019-11245: v1.14.2, v1.13.6: container uid [...] (see https://github.com/kubernetes/kubernetes/issues/78308)
+    # - CVE-2019-11250: TOB-K8S-001: Bearer tokens [...] (see https://github.com/kubernetes/kubernetes/issues/81114)
+    # We don't know if there are going to be version numbers and/or vendor IDs but the description should be last.
+    title = title.split(': ')
+    if len(title) > 0:
+        cve['summary'] = title[-1]
+        if len(title) > 1:
+            cve_id = title[0]
+            cve['id'] = cve_id
+            cve['external_url'] = f'https://www.cve.org/cverecord?id={cve_id}'
+            cve['_kubernetes_io']['google_group_url'] = f'https://groups.google.com/g/kubernetes-announce/search?q={cve_id}'
     cve_list.append(cve)
-cves = json.dumps(cve_list, sort_keys=True, indent=4)
-print(cves)
-# write the final cve list to official_cve_feed.json
-with open("official-cve-feed.json", "w") as cvejson:
-    cvejson.write(cves)
+
+feed_envelope['items'] = cve_list
+json_feed = json.dumps(feed_envelope, sort_keys=False, indent=4)
+print(json_feed)