cve_markdown_charts.py

import aiohttp
import asyncio
import requests
from datetime import datetime
import json
from mdutils.tools.Table import Table
import argparse
import dateparser
import html
from pathlib import Path

from cvedata import msrc_cvrf
from cvedata import cwe as cvedata_cwe
from cvedata import chromerelease
from cvedata import nist

# https://nvd.nist.gov/vuln/data-feeds#JSON_FEED
NIST_API_URL = "https://services.nvd.nist.gov/rest/json/cves/1.0/"
#github_raw_json_url = "https://raw.githubusercontent.com/CVEProject/cvelist/master/"
NIST_MAX_RESULTS = 2000

OUTPUT_PATH = Path('charts')
OUTPUT_PATH.mkdir(exist_ok=True, parents=True)

ARGS_CACHE = Path('.args-cache.json')

NIST_API_KEY = ''

def trim_cve_description(desc):
    """ Trim CVE desciptions"""
    splitter = "This CVE ID is unique"
    if splitter in desc:
        desc = desc.split(splitter)[0]
    return desc

def esc_mermaid(text):
    text = html.escape(text,True)
    text = text.replace(':','-')
    return text

def get_cve_list_from_keyword_ndist(keywords, strict=False):

    cve_list = []

    if strict:
        strict_param = '&isExactMatch=True'
    else: 
        strict_param = ''

    if NIST_API_KEY:
        api_key = f"&apiKey={NIST_API_KEY}"        
    else:
        api_key = ''

    for keyword in keywords:
        # handle multiple requests (max 2000 per request)
        url = f"{NIST_API_URL}?keyword={keyword}&resultsPerPage={NIST_MAX_RESULTS}{strict_param}{api_key}"

        print(f"Requesting {url}...")
        response = requests.get(url)
        count = 0
        
        while (response.status_code == 200):

            data = json.loads(response.content)
            print(f"Totals results {data['totalResults']}")
            count += len(data['result']['CVE_Items'])
            print(f"Current received results {count}")

            # create list of CVEs
            for cve in data['result']['CVE_Items']:
                cve_list.append(cve)

            if count < data['totalResults']:
                next_url = url + f"&startIndex={count}"
                response = requests.get(next_url)
            else:
                break

    # sort list by date
    return sorted(cve_list, key=lambda x: x['publishedDate'], reverse=True)

def get_cve_list_from_cve_id_list_ndist(cve_id_list):
    cve_list = []

    # clear out duplicates
    cve_id_list = list(set(cve_id_list))

    cve_list = nist.get_cves(cve_id_list)

    # clean out None cve results
    cve_list = [cve for cve in cve_list if cve]

    return cve_list

def trim_cve_list_by_date(cve_list, start, end):

    if not start and not end:
        return cve_list

    trimmed_cve_list = []

    start = dateparser.parse(str(start), settings={'RETURN_AS_TIMEZONE_AWARE': True})
    end = dateparser.parse(str(end), settings={'RETURN_AS_TIMEZONE_AWARE': True})

    for cve in cve_list:
        # filter by date
        pubDate = dateparser.parse(cve['publishedDate'])
        if not start and pubDate <= end or not end and pubDate >= start or pubDate >= start and pubDate <= end:
            trimmed_cve_list.append(cve)
        else:
            print("Skipping {} with pubDate:{}".format(
                cve['cve']['CVE_data_meta']['ID'], cve['publishedDate']))

    return trimmed_cve_list


def get_cve_list_from_cvrf_id(cvrf_ids):

    cve_list = []

    for cvrf_id in cvrf_ids:
        cvrf_json = msrc_cvrf.get_knowledge_base_cvrf_json(cvrf_id)

        if cvrf_json:
            cve_id_list = [vuln["CVE"] for vuln in cvrf_json["Vulnerability"]]

            cve_list.extend(get_cve_list_from_cve_id_list_ndist(cve_id_list))

            # sort list by date
            cve_list = sorted(
                cve_list, key=lambda x: x['publishedDate'], reverse=True)

    return cve_list


def get_cve_list_from_cvrf_tag(tags):
    cve_list = []

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    cve_id_list = []

    for tag in tags:

        for cvrf_json in msrc_cvrf_json:
            if cvrf_json.get("Vulnerability"):
                [cve_id_list.append(vuln["CVE"]) for vuln in cvrf_json["Vulnerability"] for note in vuln['Notes']
                 if note['Type'] == 7 and note.get('Value') and tag.lower() in note.get('Value').lower()]

    cve_list = get_cve_list_from_cve_id_list_ndist(cve_id_list)

    # sort list by date
    return sorted(cve_list, key=lambda x: x['publishedDate'], reverse=True)


def get_cve_list_from_windows_build(builds):
    cve_list = []

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    cve_id_list = []

    for build in builds:

        for cvrf_json in msrc_cvrf_json:
            if cvrf_json.get("Vulnerability"):
                [cve_id_list.append(vuln["CVE"]) for vuln in cvrf_json["Vulnerability"]
                 for rems in vuln["Remediations"] if rems.get('FixedBuild') and build == rems.get('FixedBuild')]

    cve_list = get_cve_list_from_cve_id_list_ndist(cve_id_list)

    # sort list by date
    return sorted(cve_list, key=lambda x: x['publishedDate'], reverse=True)


def get_cve_list_from_KB(kbs):
    cve_list = []

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    cve_id_list = []

    for kb in kbs:
        kb = kb.lower().replace('kb', '')
        for cvrf_json in msrc_cvrf_json:
            if cvrf_json.get("Vulnerability"):
                [cve_id_list.append(vuln["CVE"]) for vuln in cvrf_json["Vulnerability"] for kbs in vuln["Remediations"] if kbs['Description'].get(
                    'Value') and (str(kbs['Description']['Value']).isnumeric()) and kb in str(kbs['Description']['Value'])]

    cve_list = get_cve_list_from_cve_id_list_ndist(cve_id_list)

    # sort list by date
    return sorted(cve_list, key=lambda x: x['publishedDate'], reverse=True)


def get_cve_list_from_researcher(researchers):

    cve_list = []

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    cve_id_list = []

    for researcher in researchers:

        # Query acknowledgements from MSRC
        for cvrf_json in msrc_cvrf_json:
            if cvrf_json.get("Vulnerability"):
                [cve_id_list.append(vuln["CVE"]) for vuln in cvrf_json["Vulnerability"] for acks in vuln["Acknowledgments"]
                    for ack in acks['Name'] if ack.get('Value') is not None and researcher.lower() in ack.get('Value').lower()]

    cve_list = get_cve_list_from_cve_id_list_ndist(cve_id_list)

    # sort list by date
    cve_list = sorted(cve_list, key=lambda x: x['publishedDate'], reverse=True)

    return cve_list


def build_markdown_table_from_cves(cves, keyword):
    print("Building table...")

    table_list = []
    table_list.extend(['CVE', 'Description', 'Release Date',
                      'KBs', 'Acknowledgments', 'References', 'CNA'])
    column_len = len(table_list)

    for cve in cves:
        cve_id = cve['cve']['CVE_data_meta']['ID']
        print(cve_id)
        cve_description = trim_cve_description(cve['cve']['description']['description_data'][0]['value'])
        refs = [ref['url']
                for ref in cve['cve']['references']['reference_data']]
        cna = cve['cve']['CVE_data_meta']['ASSIGNER']

        release_date = datetime.strptime( 
            cve['publishedDate'], '%Y-%m-%dT%H:%MZ')

        # enrich with available data

        # cheat a bit here - assume year month matches cvrf
        cvrf_id = release_date.strftime("%Y-%b")
        release_date = release_date.strftime("%Y-%m-%d")
        cvrf_json = msrc_cvrf.get_knowledge_base_cvrf_json(cvrf_id)

        if cvrf_json:
            release_date = '[{}](https://msrc.microsoft.com/update-guide/en-US/vulnerability/{})'.format(
                release_date, cve_id)
            kbs = sorted(['[{}]({}) - [KB{}]({})'.format(kb.get('FixedBuild'), 'https://support.microsoft.com/help/{}'.format(kb['Description']['Value']), kb['Description']['Value'], kb['URL'])
                         for vuln in cvrf_json["Vulnerability"] if vuln["CVE"] == cve_id for kb in vuln["Remediations"] if kb['Description'].get('Value') and (str(kb['Description']['Value']).isnumeric() and 'catalog' in kb['URL'])])
            acks = {'{}'.format(ack['Name'][0].get('Value')) for vuln in cvrf_json["Vulnerability"]
                    if vuln["CVE"] == cve_id for ack in vuln["Acknowledgments"]}
        else:
            kbs = ''
            builds = ''
            acks = ''

        cve_link = '[{}](https://www.cve.org/CVERecord?id={})'.format(cve_id, cve_id)

        table_list.extend([cve_link, cve_description, release_date, '<details>'+'<br>'.join(
            kbs)+'</details>', '<br>'.join(acks).replace('\n', ' '), '<br>'.join(refs), cna])

    cve_table = Table().create_table(columns=column_len, rows=len(
        cves)+1, text=table_list, text_align='center')

    # write results to disk
    table_path = OUTPUT_PATH / (keyword.replace(' ', '-') + '-table.md')
    table_path.write_text(cve_table, encoding='UTF-8')

    print(cve_table)
    


def build_markdown_gantt_from_cves_by_release_date(cves, keyword='CVE Markdown Gantt'):
    class_template = '''
```mermaid
classDiagram

{rows}
'''


    gantt_template = '''
```mermaid
gantt

title {keyword}
dateFormat YYYY-MM-DD
axisFormat %Y-%m

section CVE Release Dates
{rows}

```
'''

    print("Building gantt chart...")

    rows = []

    sections = {}

    tag_sections = {}
    tag_rows = []
    tag_flow_rows = []

    for num, cve in enumerate(cves):

        cve_id = cve['cve']['CVE_data_meta']['ID']
        release_date = datetime.strptime(
            cve['publishedDate'], '%Y-%m-%dT%H:%MZ')
        fake_cvrf_id = release_date.strftime("%Y-%b")
        release_date = release_date.strftime("%Y-%m-%d")
        tag = get_tag_from_cve(cve_id)
        if not tag:
            tag = "None"

        row = '{} :cve{}, {}, 30d'.format(cve_id, num, release_date)
        sections.setdefault(fake_cvrf_id, []).append(row)

        tag_sections.setdefault(tag, []).append(row)

    sorted_sections = sorted(
        sections.items(), key=lambda x: datetime.strptime(x[0], '%Y-%b'), reverse=True)

    # sorted_tag_sections = sorted(
    #    tag_sections.items(), key=lambda x: len(x[1]), reverse=True)

    for section in sorted_sections:
        rows.append('section {}'.format(esc_mermaid(section[0])))
        rows.append('\n'.join(section[1]))
    #sanitize sections
    for section in tag_sections.items():
        tag_rows.append('section {}'.format(esc_mermaid(section[0])))
        tag_rows.append('\n'.join(section[1]))
        tag_flow_rows.append('class {}{{'.format(esc_mermaid(section[0])))
        section_mod = []
        for cve1 in section[1]:
            section_mod.append(cve1.split()[0])
        tag_flow_rows.append("\n".join(section_mod))
        tag_flow_rows.append('}')
        
    gantt = ''    
    gantt = gantt_template.format(keyword=keyword, rows='\n'.join(rows))
    gantt += gantt_template.format(keyword=keyword, rows='\n'.join(tag_rows))
    gantt += class_template.format(keyword=keyword, rows='\n'.join(tag_flow_rows))


    gantt_path = OUTPUT_PATH / Path(keyword.replace(' ', '-') + '-gantt.md')
    gantt_path.write_text(gantt, encoding='UTF-8')

    print(gantt)
    print(f'Gantt chart available: {gantt_path}')


def build_markdown_gantt_researcher_vanity_chart(cves, researcher):
    return None


def build_markdown_pie_researcher_vanity_cwe_chart(cves, researcher):
    return None


def build_markdown_pie_from_cves_by_cwe(cves, keyword):
    pie_template = '''
```mermaid
pie showData
    title {keyword}
{rows}

```
'''

    print("Building pie chart...")

    rows = []

    table_rows = []
    table_list = []
    table_list.extend(['CWE', 'Description', 'CVEs', 'Count'])
    column_len = len(table_list)
    totalCVEs = 0
    totalCWEs = 0

    cwes = {}
    cnas = {}
    products = []
    vendors = []
    cpes = []

    for num, cve in enumerate(cves):
        cve_id = cve['cve']['CVE_data_meta']['ID']
        totalCVEs += 1

        cna = cve['cve']['CVE_data_meta']['ASSIGNER']
        cnas.setdefault(cna, []).append(cve_id)

        problems = cve['cve']['problemtype']['problemtype_data']
        for problem in problems:
            totalCWEs += 1
            # assuming there is only ever 1 assigned?
            if len(problem['description']) > 0:
                cwe = problem['description'][0]['value']
                cwes.setdefault(cwe, []).append(cve_id)

        nodes = cve['configurations']['nodes']
        for node in nodes:
            for cpe in node['cpe_match']:
                print(cpe['cpe23Uri'])
                vendor = cpe['cpe23Uri'].split(':')[3]
                product = cpe['cpe23Uri'].split(':')[4]
                products.append(product)
                vendors.append(vendor)
                cpes.append(':'.join(cpe['cpe23Uri'].split(':')[3:5]))

    # sort dict by length of CVEs per CWE key
    sorted_cwes = {k: cwes[k] for k in sorted(
        cwes, key=lambda x: len(cwes[x]), reverse=True)}

    sorted_cnas = {k: cnas[k] for k in sorted(
        cnas, key=lambda x: len(cnas[x]), reverse=True)}

    cna_rows = []
    for cna in sorted_cnas:
        cna_rows.append('    "{}" : {}'.format(cna, len(sorted_cnas[cna])))

    cna_pie = pie_template.format(
        keyword=keyword + '- CNA Distribution', rows='\n'.join(cna_rows))

    cpe_rows = []
    keys = set(cpes)

    for cpe in keys:
        cpe_rows.append('    "{}" : {}'.format(cpe, cpes.count(cpe)))

    cpe_rows = sorted(cpe_rows, key=lambda x: int(
        x.split(':')[2]), reverse=True)
    cpe_pie = pie_template.format(
        keyword=keyword + '- CPE Distribution', rows='\n'.join(cpe_rows))

    cwe_json = cvedata_cwe.get_cwe_json()

    max_pie_piece = 15
    leftovers_count = 0
    count = 0
    for cwe in sorted_cwes:
        # build markdown pie rows
        if count <= max_pie_piece:
            rows.append('    "{}" : {}'.format(cwe, len(cwes[cwe])))
        else:
            leftovers_count += len(cwes[cwe])

        # lookup cwe
        cwe_id = str(cwe.split('-')[1])
        if cwe_id[0].isnumeric():  # handle NoInfo case
            cwe_name = cwe_json.get(cwe_id)['Name']
            cwe_url = "[{}](https://cwe.mitre.org/data/definitions/{}.html)".format(
                cwe, cwe.split('-')[1])
        else:
            cwe_name = None
            cwe_url = "{}".format(cwe)

        table_rows.append([cwe_url, cwe_name, '<details>' +
                          '<br>'.join(cwes[cwe])+'</details>', len(cwes[cwe])])
        count += 1

    for row in table_rows:
        table_list.extend(row)

    cwe_table = Table().create_table(columns=column_len, rows=len(
        table_rows)+1, text=table_list, text_align='center')

    # add leftovers other row if needed
    if leftovers_count > 0:
        rows.append('    "{}" : {}'.format('Leftovers', leftovers_count))

    pie = pie_template.format(
        keyword=keyword + '- CWE Distribution', rows='\n'.join(rows))
    pie += cwe_table
    pie += cna_pie
    pie += cpe_pie

    pie_path = OUTPUT_PATH / Path(keyword.replace(' ', '-') + '-pie.md')
    pie_path.write_text(pie, encoding='UTF-8')

    print(pie)
    print(f'Pie chart available: {pie_path}')

    return None


def get_tag_from_cve(cve_id):

    print(cve_id)

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    tag = []

    for cvrf in msrc_cvrf_json:

        if not cvrf.get('Vulnerability'):
            continue

        # skip if years don't match
        if cve_id.split('-')[1] != cvrf['DocumentTracking']['Identification']['ID']['Value'].split('-')[0]:
            continue

        [tag.append(note.get('Value')) for vuln in cvrf['Vulnerability'] if vuln['CVE']
         == cve_id for note in vuln['Notes'] if note['Type'] == 7 and note.get('Value')]

    assert(len(tag) <= 1)

    return tag[0] if len(tag) == 1 else None


def build_pie_table_combo_from_dict_by_eval_func(dict_to_sort, sort_func, table_header_list=['key', 'value', 'count'], title='Default Title', max_pie_display=15, hide_details=True):
    pie_template = '''
```mermaid
pie showData
    title {keyword}
{rows}
```
'''

    # sort dict by sort_func
    sorted_d = {k: dict_to_sort[k] for k in sorted(
        dict_to_sort, key=lambda x: sort_func(dict_to_sort[x]), reverse=True)}

    table_rows = []
    table_list = []
    table_list.extend(table_header_list)
    column_len = len(table_list)

    rows = []

    for i, key in enumerate(sorted_d):
        if i > max_pie_display:
            continue
        if i <= max_pie_display:
            rows.append('    "{}" : {}'.format(key, sort_func(sorted_d[key])))

        if hasattr(sorted_d[key], '__iter__'):
            if hide_details:
                table_rows.append(
                    [key, '<details>'+'<br>'.join(sorted_d[key])+'</details>', sort_func(sorted_d[key])])
            else:
                table_rows.append(
                    [key, '<br>'.join(sorted_d[key]), sort_func(sorted_d[key])])
        else:
            table_rows.append(
                [key, sorted_d[key], sort_func(sorted_d[key])])

    for row in table_rows:
        table_list.extend(row)

    table = Table().create_table(columns=column_len, rows=len(
        table_rows)+1, text=table_list, text_align='center')

    # build pie md

    pie = pie_template.format(
        keyword=title, rows='\n'.join(rows))

    return pie + table


def build_markdown_goat_charts_from_chrome_data():

    chromerelease_cve_json = chromerelease.get_chromerelease_cve_json()

    cves_type = {}
    cves_component = {}
    goat_chrome_researcher = {}
    goat_reward_researcher = {}

    for cve in chromerelease_cve_json:
        if not cve.get('cve_id'):
            continue

        # All Time CVEs by type
        if cve['type']:
            cves_type.setdefault(cve['type'], []).append(cve['cve_id'])

        # All Time CVE by component
        if cve['component']:
            cves_component.setdefault(
                cve['component'], []).append(cve['cve_id'])

        # The GOATs
        if cve['acknowledgment']:
            goat_chrome_researcher.setdefault(
                cve['acknowledgment'], []).append(cve['cve_id'])

            # only is reward is known
            if cve['reward'] and cve['reward'].isnumeric():
                goat_reward_researcher.setdefault(cve['acknowledgment'], 0)
                goat_reward_researcher[cve['acknowledgment']
                                       ] += int(cve['reward'])

    type_md = build_pie_table_combo_from_dict_by_eval_func(cves_type, lambda x: len(
        x), ['CVE Type', 'CVEs', 'Count'], 'All Time Chrome CVE Data by Type', 15)
    component_md = build_pie_table_combo_from_dict_by_eval_func(cves_component, lambda x: len(
        x), ['Chrome Component', 'CVEs', 'Count'], 'All Time Chrome CVE Data by Component', 15)
    goat_md = build_pie_table_combo_from_dict_by_eval_func(goat_chrome_researcher, lambda x: len(
        x), ['GOAT CVE Researcher', 'CVEs', 'Count'], 'GOAT Chrome Researcher', 15)
    reward_goat_md = build_pie_table_combo_from_dict_by_eval_func(goat_reward_researcher, lambda x: x, [
                                                                  'GOAT $$$ Researcher', 'CVEs', 'Count'], 'GOAT $$$ Chrome  Researcher', 15)

    chrome_all_path = OUTPUT_PATH / Path('chrome-all-data-charts.md')
    chrome_all_path.write_text(type_md + component_md + goat_md + reward_goat_md, encoding='UTF-8')
    
    return

# TODO build data similar to chrome


def build_markdown_goat_charts_from_msrc_cvrf_data():
    return None

    # tags = {}
    # impact = {}

    # cvrf_id = None

    # for cvrf in msrc_cvrf_json:

    #     if not cvrf.get('Vulnerability'):
    #         continue

    #     if cvrf_id:
    #         if cvrf_id != cvrf['DocumentTracking']['Identification']['ID']['Value']:
    #             continue
    #     else:
    #         # build chart with all data
    #         pass

    #     [tags.setdefault(note.get('Value'), []).append(vuln['CVE'])
    #      for vuln in cvrf['Vulnerability'] for note in vuln['Notes'] if note['Type'] == 7]
    #     [impact.setdefault(threat['Description'].get('Value'), []).append(vuln['CVE'])
    #      for vuln in cvrf['Vulnerability'] for threat in vuln['Threats'] if threat['Type'] == 0]


def build_markdown_pie_by_cves_from_cvrf_data(cve_list, title):
    print("Building cvrf pie chart...")

    msrc_cvrf_json = msrc_cvrf.get_msrc_merged_cvrf_json()

    tags = {}
    impact = {}
    missing = []

    for cve in cve_list:
        cve_id = cve['cve']['CVE_data_meta']['ID']
        cve_found = None

        for cvrf in msrc_cvrf_json:

            if not cvrf.get('Vulnerability'):
                continue

            for vuln in cvrf['Vulnerability']:
                if vuln['CVE'] == cve_id:
                    cve_found = cve_id
            [tags.setdefault(note.get('Value'), []).append(vuln['CVE']) for vuln in cvrf['Vulnerability']
             if vuln['CVE'] == cve_id for note in vuln['Notes'] if note['Type'] == 7]
            [impact.setdefault(threat['Description'].get('Value'), set()).add(
                vuln['CVE']) for vuln in cvrf['Vulnerability'] if vuln['CVE'] == cve_id for threat in vuln['Threats'] if threat['Type'] == 0]

        if not cve_found:
            missing.append(cve_id)

    print("The following CVEs were not found in CVRF data {}".format(missing))

    tags_md = build_pie_table_combo_from_dict_by_eval_func(tags, lambda x: len(
        x), ['Tag', 'CVEs', 'Count'], 'Windows Tags Distribution', 20)
    impact_md = build_pie_table_combo_from_dict_by_eval_func(impact, lambda x: len(
        x), ['Impact', 'CVEs', 'Count'], 'Windows Impact Distribution', 20)

    # sorted_tags = {k: tags[k] for k in sorted(
    #     tags, key=lambda x: len(tags[x]), reverse=True)}

    # # set is needed here due to 1 to many relationship of cve to impact
    # sorted_impact = {k: set(impact[k]) for k in sorted(
    #     impact, key=lambda x: len(impact[x]), reverse=True)}

    # pie = ''
    # keyword = title

    # table_rows = []
    # table_list = []
    # table_list.extend(['tag', 'CVEs', 'Count'])
    # column_len = len(table_list)
    # tag_rows = []

    # max_pie_display = 20
    # leftovers = []

    # for i, tag in enumerate(sorted_tags):
    #     if i <= max_pie_display:
    #         tag_rows.append('    "{}" : {}'.format(tag, len(sorted_tags[tag])))
    #     # else:
    #     #     leftovers.append([tag,len(sorted_tags[tag])])

    #     table_rows.append(
    #         [tag, '<details>'+'<br>'.join(sorted_tags[tag]), len(sorted_tags[tag])])

    # # if len(leftovers) > 0:
    # #     #tag_rows.append('    "{}" : {}'.format("leftovers", leftovers))
    # #     #print(leftovers)

    # for row in table_rows:
    #     table_list.extend(row)

    # tag_table = Table().create_table(columns=column_len, rows=len(
    #     table_rows)+1, text=table_list, text_align='center')

    # # build pie md
    # tag_pie = pie_template.format(
    #     keyword="Top {} Windows Tags Distribution - {}".format(max_pie_display, keyword), rows='\n'.join(tag_rows))
    # pie += tag_pie
    # pie += tag_table

    # table_rows = []
    # table_list = []
    # table_list.extend(['Impact', 'CVEs', 'Count'])
    # column_len = len(table_list)

    # impact_rows = []
    # max_pie_display = 20

    # for i, impact in enumerate(sorted_impact):
    #     if i <= max_pie_display:
    #         impact_rows.append('    "{}" : {}'.format(
    #             impact, len(sorted_impact[impact])))

    #     table_rows.append(
    #         [impact, '<details>'+'<br>'.join(sorted_impact[impact]), len(sorted_impact[impact])])

    # for row in table_rows:
    #     table_list.extend(row)

    # impact_table = Table().create_table(columns=column_len, rows=len(
    #     sorted_impact)+1, text=table_list, text_align='center')

    # # build pie md
    # impact_pie = pie_template.format(
    #     keyword='Impact Distribution - ' + keyword, rows='\n'.join(impact_rows))

    # pie += tags_md
    # pie += impact_md

    pie_path = OUTPUT_PATH / Path(title.replace(' ', '-') + '-tags-impact-pie.md')

    pie_path.write_text(tags_md + impact_md, encoding='UTF-8')

    return None

parser = argparse.ArgumentParser(description='Generate CVE Markdown Charts')

parser.add_argument('keyword', action='append', nargs='+',
                    help='The CVE keyword to chart (default)', default=None)
parser.add_argument('--keyword',  action='append', nargs='+',
                    help='Additional CVE keywords to chart')
parser.add_argument('--title',  action='append', nargs='+',
                    help='Set default chart title')
parser.add_argument('--output-path',  action='store', help=f'Set output path for charts. Default "./{OUTPUT_PATH}"', default=f"{OUTPUT_PATH}")
parser.add_argument('--batch-args',  action='store', help='Path to list of arguments for batch processing')
        

group = parser.add_mutually_exclusive_group()
group.add_argument('--researcher',  action='store_true',
                   help='Keyword= The researcher to chart (aka Researcher Vanity Charts)')
group.add_argument('--cvelist', action='store_true',
                   help='Keyword= List of CVEs to chart. Space separated. ex: "CVE-2022-1234 CVE-2022-1235"')
group.add_argument('--kb',  action='store_true',
                   help='Keyword= The KB Article to chart (Windows)')
group.add_argument('--winbuild',  action='store_true',
                   help='Keyword= The Windows Build Number to chart (Windows)')
group.add_argument('--cvrfid', action='store_true',
                   help='Keyword= The MSRC Security Update to chart. "Apr-2022" (Windows)')
group.add_argument('--cvrftag', action='store_true',
                   help='Keyword= Specific MSRC CVRF "tag" to chart. "Remote Procedure Call" or "Windows SMB"')
group.add_argument('--chromeid', action='store_true',
                   help='Keyword= Specific Google Release Blog Year-Month to chart. "2022-05" or "2021-01"')
group.add_argument('--chromeall', action='store_true',
                   help='Create All Time Chrome Summary Charts')
group.add_argument('--msrcall', action='store_true',
                   help='Create All Time MSRC Summary Charts')                   

group = parser.add_argument_group('CVE List Restrictions')
group.add_argument('--start', type=str, nargs='+',
                   help='Start date for CVE published. "3 years ago" or "2020/02/02"')  # CVE started in 1999
group.add_argument('--end', type=str, nargs='+',
                   help='End date for CVE published. "now" or "2020/02/02"', default='Now')

args = parser.parse_args()

print(args)

args_list = []
cve_list = None

if args.batch_args:
    # load from batch args list
    arg_list_path = Path(args.batch_args)
    args_list = json.loads(arg_list_path.read_text())
else:
    # load args from command line parse
    args_list.append(args.__dict__)

if ARGS_CACHE.exists():
    args_cache_list = json.loads(ARGS_CACHE.read_text())
else:
    args_cache_list = []


for dict_arg in args_list:

    # convert args back to Namespace    
    args = parser.parse_args()
    args.__dict__ = dict_arg

    keywords = [' '.join(word).strip() for word in args.keyword]

    if args.title:
        title = args.title
    else:
        title = '-'.join(keywords)

    OUTPUT_PATH = Path(args.output_path)

    print(keywords)

    # # Get List of CVEs
    if args.cvelist:
        # CVE List
        cve_id_list = set(' '.join(keywords).split(' '))
        title = "CVE list - {} CVEs Total".format(len(cve_id_list))
        cve_list = get_cve_list_from_cvedata_nist(cve_id_list)
    elif args.researcher:
        # Researcher
        cve_list = get_cve_list_from_researcher(keywords)
    elif args.kb:
        # KB Article
        cve_list = get_cve_list_from_KB(keywords)
    elif args.winbuild:
        # Windows Build Number
        cve_list = get_cve_list_from_windows_build(keywords)
    elif args.cvrfid:
        # MSRC CVRF
        cve_list = get_cve_list_from_cvrf_id(keywords)
    elif args.cvrftag:
        cve_list = get_cve_list_from_cvrf_tag(keywords)
    elif args.chromeall:
        build_markdown_goat_charts_from_chrome_data()
    elif args.msrcall:
        build_markdown_goat_charts_from_msrc_cvrf_data()
    else:
        # CVE keyword
        cve_list = get_cve_list_from_keyword_ndist(keywords)


    if cve_list:
        # Trim list by date
        print("Processing CVE list with len {}.".format(len(cve_list)))
        cve_list = trim_cve_list_by_date(cve_list, args.start, args.end)

        build_markdown_table_from_cves(cve_list, title)
        build_markdown_gantt_from_cves_by_release_date(cve_list, title)
        build_markdown_pie_from_cves_by_cwe(cve_list, title)
        build_markdown_pie_by_cves_from_cvrf_data(cve_list, title)


    # cache completed arg
    args_cache_list.append(args.__dict__)
    with ARGS_CACHE.open('w') as f:
        f.write(json.dumps(args_cache_list,indent=4))