From 8fae041fb20aaf36223be6290a22ce387f517b23 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 09:48:31 +0100 Subject: [PATCH 01/24] Use Maven dependency name as coordinate --- exasol/toolbox/tools/security.py | 4 ++-- test/unit/security_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py index 914ba86b1..22f79fe0e 100644 --- a/exasol/toolbox/tools/security.py +++ b/exasol/toolbox/tools/security.py @@ -88,14 +88,14 @@ def from_maven(report: str) -> Iterable[Issue]: # Note: Consider adding warnings if there is the same cve with multiple coordinates report = json.loads(report) dependencies = report.get("vulnerable", {}) # type: ignore - for _, dependency in dependencies.items(): # type: ignore + for dependency_name, dependency in dependencies.items(): # type: ignore for v in dependency["vulnerabilities"]: # type: ignore references = [v["reference"]] + v["externalReferences"] yield Issue( cve=v["cve"], cwe=v["cwe"], description=v["description"], - coordinates=dependency["coordinates"], + coordinates=dependency_name, references=tuple(references), ) diff --git a/test/unit/security_test.py b/test/unit/security_test.py index 80583e90f..5aea0ecc7 100644 --- a/test/unit/security_test.py +++ b/test/unit/security_test.py @@ -317,7 +317,7 @@ def test_convert_maven_input(maven_report): # pylint: disable=redefined-outer-n "Java SDK up to and including 1.11.2. Users should update " "to apache-avro version 1.11.3 which addresses this issue.\n" "\n", - coordinates="pkg:maven/org.apache.avro/avro@1.7.7", + coordinates="org.apache.avro:avro:jar:1.7.7:compile", references=( "https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", @@ -344,7 +344,7 @@ def test_convert_maven_input(maven_report): # pylint: disable=redefined-outer-n "differ from those defined at NVD. See " "https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 " "for details", - coordinates="pkg:maven/fr.turri/aXMLRPC@1.13.0", + coordinates="fr.turri:aXMLRPC:jar:1.13.0:test", references=( "https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", From 70836479460ac3b9410dbb267978bc8761b9ab52 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 09:52:44 +0100 Subject: [PATCH 02/24] Output created file --- .github/actions/security-issues/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 8241e99d7..0f59cac64 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -59,6 +59,9 @@ runs: run: | tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.txt + - name: Cat info + run: cat created.txt + - name: Create Report shell: bash run: | From 6420d0b624be72ec6edbe864a862697e51c54bdd Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 09:54:35 +0100 Subject: [PATCH 03/24] Add required shell --- .github/actions/security-issues/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 0f59cac64..45cc8fb76 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -60,6 +60,7 @@ runs: tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.txt - name: Cat info + shell: bash run: cat created.txt - name: Create Report From c6753e3e8387e117ced199fe6176d8ca6a8459ac Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 09:57:13 +0100 Subject: [PATCH 04/24] Upgrade setup-python --- .github/actions/python-environment/action.yml | 4 ++-- .github/actions/security-issues/action.yml | 2 +- .../templates/github/actions/python-environment/action.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/python-environment/action.yml b/.github/actions/python-environment/action.yml index d968434b1..953071760 100644 --- a/.github/actions/python-environment/action.yml +++ b/.github/actions/python-environment/action.yml @@ -19,7 +19,7 @@ runs: steps: - name: Setup Python (${{ inputs.python-version}}) - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} @@ -30,4 +30,4 @@ runs: - name: Poetry install run: poetry install - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 45cc8fb76..717a7961d 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -25,7 +25,7 @@ runs: steps: - name: Setup Python (${{ inputs.python-version}}) - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/exasol/toolbox/templates/github/actions/python-environment/action.yml b/exasol/toolbox/templates/github/actions/python-environment/action.yml index 839e6316d..953071760 100644 --- a/exasol/toolbox/templates/github/actions/python-environment/action.yml +++ b/exasol/toolbox/templates/github/actions/python-environment/action.yml @@ -19,7 +19,7 @@ runs: steps: - name: Setup Python (${{ inputs.python-version}}) - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} From 108988fd04823892774d52cfe1aa5e73269b55da Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 10:34:20 +0100 Subject: [PATCH 05/24] Output information about created issues --- .github/actions/security-issues/action.yml | 19 +++++++++++++++---- exasol/toolbox/tools/security.py | 10 ++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 717a7961d..d4627650f 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -19,6 +19,13 @@ inputs: description: 'Project the created tickets shall be associated with' required: false + +outputs: + + created-issues: + description: 'Information about the created issues and the vulnerabilities in JSONL format' + value: ${{ steps.get-created-issues.outputs.issues }} + runs: using: "composite" @@ -57,17 +64,21 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.txt + tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl - - name: Cat info + - name: Define Output Parameter + id: get-created-issues shell: bash - run: cat created.txt + run: | + echo 'issues<> "$GITHUB_OUTPUT" + cat created.jsonl >> "$GITHUB_OUTPUT" + echo 'EOF' >> "$GITHUB_OUTPUT" - name: Create Report shell: bash run: | echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY echo -e "## Created Security Issue\n" >> $GITHUB_STEP_SUMMARY - cat created.txt >> $GITHUB_STEP_SUMMARY + cat created.jsonl >> $GITHUB_STEP_SUMMARY echo -e "## Filtered Security Issue\n" >> $GITHUB_STEP_SUMMARY tail -n +2 filtered.txt | grep . || true >> $GITHUB_STEP_SUMMARY diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py index 22f79fe0e..e631147b5 100644 --- a/exasol/toolbox/tools/security.py +++ b/exasol/toolbox/tools/security.py @@ -250,10 +250,16 @@ def create( Output: Links to the created issue(s) """ + def _format_jsonl(issue_url: str, issue: Issue) -> str: + issue = asdict(issue) + issue["url"] = issue_url + return json.dumps(issue) + for issue in _issues(input_file): - std_err, std_out = create_security_issue(issue, project) + std_err, issue_url = create_security_issue(issue, project) stderr(std_err) - stdout(std_out) + stdout(_format_jsonl(issue_url, issue)) + if __name__ == "__main__": From 0ad77ac7eaf0acbb765b6003f11fbc357254360b Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 11:33:08 +0100 Subject: [PATCH 06/24] Run toolbox from source --- .github/actions/security-issues/action.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index d4627650f..2a7a41003 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -31,15 +31,25 @@ runs: using: "composite" steps: + - uses: actions/checkout@v4 + with: + repository: exasol/exasol-toolbox + ref: feature/#130-security-issues-output + path: toolbox + + - name: Install poetry + run: pipx install poetry + - name: Setup Python (${{ inputs.python-version}}) uses: actions/setup-python@v5 with: python-version: 3.11 + cache: 'poetry' - name: Install Python Toolbox / Security tool shell: bash run: | - pip install exasol-toolbox==0.7.0 + poetry --directory=toolbox install - name: Create Security Issue Report shell: bash @@ -49,14 +59,14 @@ runs: - name: Convert Report To Common Input Format shell: bash run: | - tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl + poetry --directory=toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl - name: Filter Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl + poetry --directory=toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl cat filtered.txt - name: Create Issues @@ -64,7 +74,7 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl + poetry --directory=toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl - name: Define Output Parameter id: get-created-issues From 705c9296fd7f2d0120ea4a270f5cc271e01aedf5 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 11:35:32 +0100 Subject: [PATCH 07/24] Fix run --- .github/actions/security-issues/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 2a7a41003..5ee4075bf 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -38,6 +38,7 @@ runs: path: toolbox - name: Install poetry + shell: bash run: pipx install poetry - name: Setup Python (${{ inputs.python-version}}) From 09547a9d797e3ac747ce5d93f49976ae6533abb7 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 11:37:28 +0100 Subject: [PATCH 08/24] Skip toolbox checkout --- .github/actions/security-issues/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 5ee4075bf..83b76f4a0 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -31,12 +31,6 @@ runs: using: "composite" steps: - - uses: actions/checkout@v4 - with: - repository: exasol/exasol-toolbox - ref: feature/#130-security-issues-output - path: toolbox - - name: Install poetry shell: bash run: pipx install poetry From 654ebb5d157b4da8cb33d3c6051efe1dded8bf99 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:20:35 +0100 Subject: [PATCH 09/24] use absolute path for poetry --- .github/actions/security-issues/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 83b76f4a0..ef019f07d 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -44,7 +44,7 @@ runs: - name: Install Python Toolbox / Security tool shell: bash run: | - poetry --directory=toolbox install + poetry --directory=$HOME/toolbox install - name: Create Security Issue Report shell: bash @@ -54,14 +54,14 @@ runs: - name: Convert Report To Common Input Format shell: bash run: | - poetry --directory=toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl + poetry --directory=$HOME/toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl - name: Filter Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl + poetry --directory=$HOME/toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl cat filtered.txt - name: Create Issues @@ -69,7 +69,7 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl + poetry --directory=$HOME/toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl - name: Define Output Parameter id: get-created-issues From dccd3e754de5e53220802e58cba067149f127a04 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:27:28 +0100 Subject: [PATCH 10/24] update path to toolbox --- .github/actions/security-issues/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index ef019f07d..f0beb7106 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -44,7 +44,7 @@ runs: - name: Install Python Toolbox / Security tool shell: bash run: | - poetry --directory=$HOME/toolbox install + poetry --directory=$GITHUB_WORKSPACE/toolbox install - name: Create Security Issue Report shell: bash @@ -54,14 +54,14 @@ runs: - name: Convert Report To Common Input Format shell: bash run: | - poetry --directory=$HOME/toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl + poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl - name: Filter Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=$HOME/toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl + poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl cat filtered.txt - name: Create Issues @@ -69,7 +69,7 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=$HOME/toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl + poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl - name: Define Output Parameter id: get-created-issues From f31a5683b2f36cb1fca3e5adbc711582cc641fba Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:34:10 +0100 Subject: [PATCH 11/24] Use relative paths for poetry --- .github/actions/security-issues/action.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index f0beb7106..d43c69735 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -54,22 +54,25 @@ runs: - name: Convert Report To Common Input Format shell: bash run: | - poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl + cd $GITHUB_WORKSPACE/toolbox + poetry run tbx security cve convert ${{inputs.format}} < ../input | tee ../cves.jsonl - name: Filter Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl - cat filtered.txt + $GITHUB_WORKSPACE/toolbox + poetry run tbx security cve filter github-issues < ../cves.jsonl 2> ../filtered.txt | tee ../issues.jsonl + cat ../filtered.txt - name: Create Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - poetry --directory=$GITHUB_WORKSPACE/toolbox run tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl + $GITHUB_WORKSPACE/toolbox + poetry run tbx security cve create --project "${{ inputs.project }}" < ../issues.jsonl | tee ../created.jsonl - name: Define Output Parameter id: get-created-issues From 518b2b4e722e47c4fc846e4d375d4bcd5cbda9b1 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:39:08 +0100 Subject: [PATCH 12/24] Move checkout back --- .github/actions/security-issues/action.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index d43c69735..79c70c4d7 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -41,6 +41,22 @@ runs: python-version: 3.11 cache: 'poetry' + - uses: actions/checkout@v4 + with: + repository: exasol/python-toolbox + ref: feature/#130-security-issues-output + path: toolbox + + - name: Debug + run: | + pwd + ls -lah + echo "GITHUB_WORKSPACE = $GITHUB_WORKSPACE" + ls -lha $GITHUB_WORKSPACE/toolbox + cd $GITHUB_WORKSPACE/toolbox + ls + ls * + - name: Install Python Toolbox / Security tool shell: bash run: | @@ -62,7 +78,7 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - $GITHUB_WORKSPACE/toolbox + cd $GITHUB_WORKSPACE/toolbox poetry run tbx security cve filter github-issues < ../cves.jsonl 2> ../filtered.txt | tee ../issues.jsonl cat ../filtered.txt @@ -71,7 +87,7 @@ runs: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - $GITHUB_WORKSPACE/toolbox + cd $GITHUB_WORKSPACE/toolbox poetry run tbx security cve create --project "${{ inputs.project }}" < ../issues.jsonl | tee ../created.jsonl - name: Define Output Parameter From 96a1e1ee6b2b789fff13eae43566230cd7674b6f Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:40:22 +0100 Subject: [PATCH 13/24] add shell --- .github/actions/security-issues/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 79c70c4d7..33410ee66 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -48,6 +48,7 @@ runs: path: toolbox - name: Debug + shell: bash run: | pwd ls -lah From 0a4d9c7453fff375da665a4e4246f78d5d49e6e7 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 14:42:58 +0100 Subject: [PATCH 14/24] Remove checkout --- .github/actions/security-issues/action.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 33410ee66..fe54143e6 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -41,23 +41,6 @@ runs: python-version: 3.11 cache: 'poetry' - - uses: actions/checkout@v4 - with: - repository: exasol/python-toolbox - ref: feature/#130-security-issues-output - path: toolbox - - - name: Debug - shell: bash - run: | - pwd - ls -lah - echo "GITHUB_WORKSPACE = $GITHUB_WORKSPACE" - ls -lha $GITHUB_WORKSPACE/toolbox - cd $GITHUB_WORKSPACE/toolbox - ls - ls * - - name: Install Python Toolbox / Security tool shell: bash run: | From 84d7478f9f570472a54c2e78f165b174565f0a95 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 1 Feb 2024 15:10:59 +0100 Subject: [PATCH 15/24] Rename GITHUB_TOKEN env variable --- .github/actions/security-issues/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index fe54143e6..09669523f 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -59,7 +59,7 @@ runs: - name: Filter Issues env: - GH_TOKEN: ${{ inputs.github-token }} + GITHUB_TOKEN: ${{ inputs.github-token }} shell: bash run: | cd $GITHUB_WORKSPACE/toolbox @@ -68,7 +68,7 @@ runs: - name: Create Issues env: - GH_TOKEN: ${{ inputs.github-token }} + GITHUB_TOKEN: ${{ inputs.github-token }} shell: bash run: | cd $GITHUB_WORKSPACE/toolbox From 67ee274bdf1b8c213a8a9984e4b958976f88f464 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 16:12:15 +0100 Subject: [PATCH 16/24] Revert change to token variable --- .github/actions/security-issues/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 09669523f..fe54143e6 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -59,7 +59,7 @@ runs: - name: Filter Issues env: - GITHUB_TOKEN: ${{ inputs.github-token }} + GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | cd $GITHUB_WORKSPACE/toolbox @@ -68,7 +68,7 @@ runs: - name: Create Issues env: - GITHUB_TOKEN: ${{ inputs.github-token }} + GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | cd $GITHUB_WORKSPACE/toolbox From 2d565e28167089a3493a39e4516a278d41569800 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 16:20:19 +0100 Subject: [PATCH 17/24] Revert changes to action --- .github/actions/security-issues/action.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index fe54143e6..e0ceedee4 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -31,20 +31,15 @@ runs: using: "composite" steps: - - name: Install poetry - shell: bash - run: pipx install poetry - - name: Setup Python (${{ inputs.python-version}}) uses: actions/setup-python@v5 with: python-version: 3.11 - cache: 'poetry' - name: Install Python Toolbox / Security tool shell: bash run: | - poetry --directory=$GITHUB_WORKSPACE/toolbox install + pip install exasol-toolbox==0.7.0 - name: Create Security Issue Report shell: bash @@ -54,26 +49,22 @@ runs: - name: Convert Report To Common Input Format shell: bash run: | - cd $GITHUB_WORKSPACE/toolbox - poetry run tbx security cve convert ${{inputs.format}} < ../input | tee ../cves.jsonl + tbx security cve convert ${{inputs.format}} < input | tee cves.jsonl - name: Filter Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - cd $GITHUB_WORKSPACE/toolbox - poetry run tbx security cve filter github-issues < ../cves.jsonl 2> ../filtered.txt | tee ../issues.jsonl - cat ../filtered.txt + tbx security cve filter github-issues < cves.jsonl 2> filtered.txt | tee issues.jsonl + cat filtered.txt - name: Create Issues env: GH_TOKEN: ${{ inputs.github-token }} shell: bash run: | - cd $GITHUB_WORKSPACE/toolbox - poetry run tbx security cve create --project "${{ inputs.project }}" < ../issues.jsonl | tee ../created.jsonl - + tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl - name: Define Output Parameter id: get-created-issues shell: bash From cb397998ea81bc3ce8b822eac701245182911cda Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 16:25:18 +0100 Subject: [PATCH 18/24] Adapt tests to new Maven coordinates --- test/integration/cli/security-cve-convert.t | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/cli/security-cve-convert.t b/test/integration/cli/security-cve-convert.t index 9ac554413..7860a00e9 100644 --- a/test/integration/cli/security-cve-convert.t +++ b/test/integration/cli/security-cve-convert.t @@ -73,7 +73,7 @@ Test convert maven input from stdin "cve": "CVE-2020-4333", "cwe": "CWE-502", "description": "When deserializing untrusted or corrupted data, it is possible for a reader to consume memory beyond the allowed constraints and thus lead to out of memory on the system.\n\nThis issue affects Java applications using Apache Avro Java SDK up to and including 1.11.2. Users should update to apache-avro version 1.11.3 which addresses this issue.\n\n", - "coordinates": "pkg:maven/org.apache.avro/avro@1.7.7", + "coordinates": "org.apache.avro:avro:jar:1.7.7:compile", "references": [ "https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", @@ -86,7 +86,7 @@ Test convert maven input from stdin "cve": "CVE-2020-36641", "cwe": "CWE-611", "description": "A vulnerability classified as problematic was found in gturri aXMLRPC up to 1.12.0. This vulnerability affects the function ResponseParser of the file src/main/java/de/timroes/axmlrpc/ResponseParser.java. The manipulation leads to xml external entity reference. Upgrading to version 1.12.1 is able to address this issue. The patch is identified as ad6615b3ec41353e614f6ea5fdd5b046442a832b. It is recommended to upgrade the affected component. VDB-217450 is the identifier assigned to this vulnerability.\n\nSonatype's research suggests that this CVE's details differ from those defined at NVD. See https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 for details", - "coordinates": "pkg:maven/fr.turri/aXMLRPC@1.13.0", + "coordinates": "fr.turri:aXMLRPC:jar:1.13.0:test", "references": [ "https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", @@ -101,7 +101,7 @@ Test convert maven input from file "cve": "CVE-2020-4333", "cwe": "CWE-502", "description": "When deserializing untrusted or corrupted data, it is possible for a reader to consume memory beyond the allowed constraints and thus lead to out of memory on the system.\n\nThis issue affects Java applications using Apache Avro Java SDK up to and including 1.11.2. Users should update to apache-avro version 1.11.3 which addresses this issue.\n\n", - "coordinates": "pkg:maven/org.apache.avro/avro@1.7.7", + "coordinates": "org.apache.avro:avro:jar:1.7.7:compile", "references": [ "https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", @@ -114,7 +114,7 @@ Test convert maven input from file "cve": "CVE-2020-36641", "cwe": "CWE-611", "description": "A vulnerability classified as problematic was found in gturri aXMLRPC up to 1.12.0. This vulnerability affects the function ResponseParser of the file src/main/java/de/timroes/axmlrpc/ResponseParser.java. The manipulation leads to xml external entity reference. Upgrading to version 1.12.1 is able to address this issue. The patch is identified as ad6615b3ec41353e614f6ea5fdd5b046442a832b. It is recommended to upgrade the affected component. VDB-217450 is the identifier assigned to this vulnerability.\n\nSonatype's research suggests that this CVE's details differ from those defined at NVD. See https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 for details", - "coordinates": "pkg:maven/fr.turri/aXMLRPC@1.13.0", + "coordinates": "fr.turri:aXMLRPC:jar:1.13.0:test", "references": [ "https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", From e1e84cdcfffc7bfe14e72779978dddd83e9c102a Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 17:01:10 +0100 Subject: [PATCH 19/24] Add unit test for formatting --- .github/actions/security-issues/action.yml | 1 + exasol/toolbox/tools/security.py | 12 +++++------- test/unit/security_test.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index e0ceedee4..d4627650f 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -65,6 +65,7 @@ runs: shell: bash run: | tbx security cve create --project "${{ inputs.project }}" < issues.jsonl | tee created.jsonl + - name: Define Output Parameter id: get-created-issues shell: bash diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py index e631147b5..4cb2945bb 100644 --- a/exasol/toolbox/tools/security.py +++ b/exasol/toolbox/tools/security.py @@ -250,17 +250,15 @@ def create( Output: Links to the created issue(s) """ - def _format_jsonl(issue_url: str, issue: Issue) -> str: - issue = asdict(issue) - issue["url"] = issue_url - return json.dumps(issue) - for issue in _issues(input_file): std_err, issue_url = create_security_issue(issue, project) stderr(std_err) - stdout(_format_jsonl(issue_url, issue)) - + stdout(format_jsonl(issue_url, issue)) +def format_jsonl(issue_url: str, issue: Issue) -> str: + issue = asdict(issue) + issue["url"] = issue_url.strip() + return json.dumps(issue) if __name__ == "__main__": CLI() diff --git a/test/unit/security_test.py b/test/unit/security_test.py index 5aea0ecc7..ab4483715 100644 --- a/test/unit/security_test.py +++ b/test/unit/security_test.py @@ -359,3 +359,14 @@ def test_convert_maven_input(maven_report): # pylint: disable=redefined-outer-n def test_convert_maven_input_no_vulnerable(): # pylint: disable=redefined-outer-name actual = set(security.from_maven("{}")) assert len(actual) == 0 + +def test_format_jsonl(): + issue = security.Issue(coordinates="coordinates", cve="cve", cwe="cwe", description="description", references=()) + actual = security.format_jsonl("issue_url", issue) + assert actual == '{"cve": "cve", "cwe": "cwe", "description": "description", "coordinates": "coordinates", "references": [], "url": "issue_url"}' + + +def test_format_jsonl_removes_newline(): + issue = security.Issue(coordinates="coordinates", cve="cve", cwe="cwe", description="description", references=()) + actual = security.format_jsonl("issue_url\n", issue) + assert actual == '{"cve": "cve", "cwe": "cwe", "description": "description", "coordinates": "coordinates", "references": [], "url": "issue_url"}' From e8d7d4e27b891dbc18c97653653a7c5827bec755 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 17:09:08 +0100 Subject: [PATCH 20/24] Adapt integration tests --- test/integration/cli/security-cve-create.t | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/cli/security-cve-create.t b/test/integration/cli/security-cve-create.t index 532171143..65ea3f5a4 100644 --- a/test/integration/cli/security-cve-create.t +++ b/test/integration/cli/security-cve-create.t @@ -2,7 +2,7 @@ Create a GitHub CLI mock $ cat > gh < #!/usr/bin/sh - > echo https://github.com/exasol/a-project/issues/`uuidgen` + > echo https://github.com/exasol/a-project/issues/123 > EOF $ chmod u+x gh @@ -19,5 +19,5 @@ Create test input Run test case $ tbx security cve create cves.jsonl 2> /dev/null | grep . - https://github.com/exasol/a-project/issues/[a-f0-9\-]+ (re) - https://github.com/exasol/a-project/issues/[a-f0-9\-]+ (re) + {"cve": "CVE-2023-39410", "cwe": "CWE-502", "description": "When deserializing untrusted or corrupted data, it is possible for a reader to consume memory beyond the allowed constraints and thus lead to out of memory on the system.\n\nThis issue affects Java applications using Apache Avro Java SDK up to and including 1.11.2. Users should update to apache-avro version 1.11.3 which addresses this issue.\n\n", "coordinates": "pkg:maven/org.apache.avro/avro@1.7.7", "references": ["https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", "https://github.com/apache/avro/pull/2432", "https://issues.apache.org/jira/browse/AVRO-3819", "https://lists.apache.org/thread/q142wj99cwdd0jo5lvdoxzoymlqyjdds"], "url": "https://github.com/exasol/a-project/issues/123"} + {"cve": "CVE-2020-36641", "cwe": "CWE-611", "description": "A vulnerability classified as problematic was found in gturri aXMLRPC up to 1.12.0. This vulnerability affects the function ResponseParser of the file src/main/java/de/timroes/axmlrpc/ResponseParser.java. The manipulation leads to xml external entity reference. Upgrading to version 1.12.1 is able to address this issue. The patch is identified as ad6615b3ec41353e614f6ea5fdd5b046442a832b. It is recommended to upgrade the affected component. VDB-217450 is the identifier assigned to this vulnerability.\n\nSonatype's research suggests that this CVE's details differ from those defined at NVD. See https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 for details", "coordinates": "pkg:maven/fr.turri/aXMLRPC@1.13.0", "references": ["https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", "https://www.tenable.com/cve/CVE-2020-36641"], "url": "https://github.com/exasol/a-project/issues/123"} From fad5c8946074a8e16c9b7642c41fa961f2e45620 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 17:16:08 +0100 Subject: [PATCH 21/24] Fix type checker --- exasol/toolbox/tools/security.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py index 4cb2945bb..32abb0be1 100644 --- a/exasol/toolbox/tools/security.py +++ b/exasol/toolbox/tools/security.py @@ -256,9 +256,9 @@ def create( stdout(format_jsonl(issue_url, issue)) def format_jsonl(issue_url: str, issue: Issue) -> str: - issue = asdict(issue) - issue["url"] = issue_url.strip() - return json.dumps(issue) + issue_json = asdict(issue) + issue_json["url"] = issue_url.strip() + return json.dumps(issue_json) if __name__ == "__main__": CLI() From f1880ad9c2076d5a80014d0458c324634d9adbab Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 2 Feb 2024 17:26:53 +0100 Subject: [PATCH 22/24] Add changelog entry --- doc/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 623478bdf..c30d79526 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -6,6 +6,9 @@ Unreleased ========== +✨ Added +-------- +* Added output `created-issues` to `security-issues` action containing the found security issues including the created GitHub issues in JSONL format. .. _changelog-0.7.0: From 365cb5226d2ab1ff017e418b78705a3fa64d116e Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 5 Feb 2024 13:17:41 +0100 Subject: [PATCH 23/24] Apply suggestions from code review Co-authored-by: Nicola Coretti --- test/unit/security_test.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/test/unit/security_test.py b/test/unit/security_test.py index ab4483715..dd8c50f3f 100644 --- a/test/unit/security_test.py +++ b/test/unit/security_test.py @@ -361,12 +361,38 @@ def test_convert_maven_input_no_vulnerable(): # pylint: disable=redefined-outer assert len(actual) == 0 def test_format_jsonl(): - issue = security.Issue(coordinates="coordinates", cve="cve", cwe="cwe", description="description", references=()) + issue = security.Issue( + coordinates="coordinates", + cve="cve", cwe="cwe", + description="description", + references=() + ) + expected = json.dumps({ + "cve": "cve", + "cwe": "cwe", + "description": "description", + "coordinates": "coordinates", + "references": [], + "url": "issue_url" + }) actual = security.format_jsonl("issue_url", issue) - assert actual == '{"cve": "cve", "cwe": "cwe", "description": "description", "coordinates": "coordinates", "references": [], "url": "issue_url"}' + assert actual == expected def test_format_jsonl_removes_newline(): - issue = security.Issue(coordinates="coordinates", cve="cve", cwe="cwe", description="description", references=()) + issue = security.Issue( + coordinates="coordinates", + cve="cve", cwe="cwe", + description="description", + references=() + ) + expected = json.dumps({ + "cve": "cve", + "cwe": "cwe", + "description": "description", + "coordinates": "coordinates", + "references": [], + "url": "issue_url" + }) actual = security.format_jsonl("issue_url\n", issue) - assert actual == '{"cve": "cve", "cwe": "cwe", "description": "description", "coordinates": "coordinates", "references": [], "url": "issue_url"}' + assert actual == expected From 361d473b8d35cf69e7f2ee8e73a31018b68d2113 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 5 Feb 2024 13:38:35 +0100 Subject: [PATCH 24/24] Format JSON assertion --- test/integration/cli/security-cve-create.t | 30 +++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/test/integration/cli/security-cve-create.t b/test/integration/cli/security-cve-create.t index 65ea3f5a4..b4eeee735 100644 --- a/test/integration/cli/security-cve-create.t +++ b/test/integration/cli/security-cve-create.t @@ -18,6 +18,30 @@ Create test input Run test case - $ tbx security cve create cves.jsonl 2> /dev/null | grep . - {"cve": "CVE-2023-39410", "cwe": "CWE-502", "description": "When deserializing untrusted or corrupted data, it is possible for a reader to consume memory beyond the allowed constraints and thus lead to out of memory on the system.\n\nThis issue affects Java applications using Apache Avro Java SDK up to and including 1.11.2. Users should update to apache-avro version 1.11.3 which addresses this issue.\n\n", "coordinates": "pkg:maven/org.apache.avro/avro@1.7.7", "references": ["https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", "https://github.com/apache/avro/pull/2432", "https://issues.apache.org/jira/browse/AVRO-3819", "https://lists.apache.org/thread/q142wj99cwdd0jo5lvdoxzoymlqyjdds"], "url": "https://github.com/exasol/a-project/issues/123"} - {"cve": "CVE-2020-36641", "cwe": "CWE-611", "description": "A vulnerability classified as problematic was found in gturri aXMLRPC up to 1.12.0. This vulnerability affects the function ResponseParser of the file src/main/java/de/timroes/axmlrpc/ResponseParser.java. The manipulation leads to xml external entity reference. Upgrading to version 1.12.1 is able to address this issue. The patch is identified as ad6615b3ec41353e614f6ea5fdd5b046442a832b. It is recommended to upgrade the affected component. VDB-217450 is the identifier assigned to this vulnerability.\n\nSonatype's research suggests that this CVE's details differ from those defined at NVD. See https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 for details", "coordinates": "pkg:maven/fr.turri/aXMLRPC@1.13.0", "references": ["https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", "https://www.tenable.com/cve/CVE-2020-36641"], "url": "https://github.com/exasol/a-project/issues/123"} + $ tbx security cve create cves.jsonl 2> /dev/null | python -m json.tool --json-lines + { + "cve": "CVE-2023-39410", + "cwe": "CWE-502", + "description": "When deserializing untrusted or corrupted data, it is possible for a reader to consume memory beyond the allowed constraints and thus lead to out of memory on the system.\n\nThis issue affects Java applications using Apache Avro Java SDK up to and including 1.11.2. Users should update to apache-avro version 1.11.3 which addresses this issue.\n\n", + "coordinates": "pkg:maven/org.apache.avro/avro@1.7.7", + "references": [ + "https://ossindex.sonatype.org/vulnerability/CVE-2023-39410?component-type=maven&component-name=org.apache.avro%2Favro&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2023-39410", + "https://github.com/apache/avro/pull/2432", + "https://issues.apache.org/jira/browse/AVRO-3819", + "https://lists.apache.org/thread/q142wj99cwdd0jo5lvdoxzoymlqyjdds" + ], + "url": "https://github.com/exasol/a-project/issues/123" + } + { + "cve": "CVE-2020-36641", + "cwe": "CWE-611", + "description": "A vulnerability classified as problematic was found in gturri aXMLRPC up to 1.12.0. This vulnerability affects the function ResponseParser of the file src/main/java/de/timroes/axmlrpc/ResponseParser.java. The manipulation leads to xml external entity reference. Upgrading to version 1.12.1 is able to address this issue. The patch is identified as ad6615b3ec41353e614f6ea5fdd5b046442a832b. It is recommended to upgrade the affected component. VDB-217450 is the identifier assigned to this vulnerability.\n\nSonatype's research suggests that this CVE's details differ from those defined at NVD. See https://ossindex.sonatype.org/vulnerability/CVE-2020-36641 for details", + "coordinates": "pkg:maven/fr.turri/aXMLRPC@1.13.0", + "references": [ + "https://ossindex.sonatype.org/vulnerability/CVE-2020-36641?component-type=maven&component-name=fr.turri%2FaXMLRPC&utm_source=ossindex-client&utm_medium=integration&utm_content=1.8.1", + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-36641", + "https://www.tenable.com/cve/CVE-2020-36641" + ], + "url": "https://github.com/exasol/a-project/issues/123" + }