Security scan pipeline update (#1711)

### Summary This PR updates security scan pipeline, in particular: - minor fixes in Trivy config to cover all deps - CodeQL is added - minor fixes in Bandit config - SARIF is used to upload results into Security tab ### How to test  ### Checklist  - [ ] I have added unit tests to cover my changes. - [ ] I have added integration tests to cover my changes. - [ ] I have added the description of my changes into [CHANGELOG](https://github.com/openvinotoolkit/datumaro/blob/develop/CHANGELOG.md). - [ ] I have updated the [documentation](https://github.com/openvinotoolkit/datumaro/tree/develop/docs) accordingly ### License - [x] I submit _my code changes_ under the same [MIT License](https://github.com/openvinotoolkit/datumaro/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. - [ ] I have updated the license header for each file (see an example below). ```python # Copyright (C) 2024 Intel Corporation # # SPDX-License-Identifier: MIT ```
openvinotoolkit · Feb 7, 2025 · 3beb5fa · 3beb5fa
2 parents ae159ff + a6f55f5
commit 3beb5fa
Show file tree

Hide file tree

Showing 3 changed files with 240 additions and 110 deletions.
diff --git a/.ci/ipas_default.config b/.ci/ipas_default.config
@@ -40,7 +40,7 @@
 # B317 : xml_bad_sax
 # B318 : xml_bad_minidom
 # B319 : xml_bad_pulldom
-# B320 : xml_bad_etree
+# B320 : xml_bad_etree - removed https://github.com/PyCQA/bandit/commit/e4da0b351f89a82b5de8dd791cbdd963476b5a11
 # B321 : ftplib
 # B323 : unverified_context
 # B324 : hashlib_new_insecure_functions
@@ -53,7 +53,7 @@
 # B407 : import_xml_expat
 # B408 : import_xml_minidom
 # B409 : import_xml_pulldom
-# B410 : import_lxml
+# B410 : import_lxml - removed https://github.com/PyCQA/bandit/commit/e4da0b351f89a82b5de8dd791cbdd963476b5a11
 # B411 : import_xmlrpclib
 # B412 : import_httpoxy
 # B413 : import_pycrypto
@@ -83,7 +83,7 @@
 # IPAS Required Checkers. Do not disable these
 # Additional checkers may be added if desired
 tests:
-  [ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B320', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B410', 'B411', 'B412', 'B413']
+  [ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B411', 'B412', 'B413']
 
 # (optional) list skipped test IDs here, eg '[B101, B406]':
 # The following checkers are not required but be added to tests list if desired

diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml
@@ -3,75 +3,224 @@ name: Code Scanning
 on:
   workflow_dispatch: # run on request (no need for PR)
   push:
-    branches: [ "develop", "releases/*" ]
+    branches: ["develop", "releases/*"]
   schedule:
     # every UTC 6PM from Mon to Fri
     - cron: "0 18 * * 1-5"
 
+permissions: {}
+
 jobs:
-  Trivy:
-    runs-on: ubuntu-latest
+  Trivy-Scan-Vuln:
+    runs-on: ubuntu-22.04
+    permissions:
+      security-events: write
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Python
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: "3.10"
+
       - name: Install dependencies
         run: python -m pip install pip-tools
+
       - name: Freeze dependencies
         run: |
-          mkdir -p .ci/base
-          pip-compile -o .ci/base/requirements.txt requirements.txt
-          mkdir -p .ci/dev/tests
-          pip-compile -o .ci/dev/tests/requirements.txt tests/requirements.txt
-          mkdir -p .ci/dev/docker/segment-anything/
-          pip-compile -o .ci/dev/docker/segment-anything/requirements.txt docker/segment-anything/requirements.txt
-          mkdir -p .ci/base/docs
-          pip-compile -o .ci/base/docs/requirements.txt docs/requirements.txt
-      - name: Run Trivy Scan (full, csv)
-        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # 0.29.0
-        with:
-          trivy-config: ".ci/trivy-csv.yaml"
-          scan-type: 'fs'
-          scan-ref: ".ci/"
-          scanners: vuln,secret
-      - name: Run Trivy Scan (prod, spdx.json)
-        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # 0.29.0
-        with:
-          trivy-config: ".ci/trivy-json.yaml"
-          scan-type: 'fs'
-          scan-ref: ".ci/base"
-      - name: Upload Trivy results
-        uses: actions/upload-artifact@v4
-        with:
-          name: trivy-results-prod-json
-          path: '${{ github.workspace }}/trivy-results-*'
+          mkdir -p trivy_input/docker/segment-anything/
+          pip-compile -o trivy_input/docker/segment-anything/requirements.txt docker/segment-anything/requirements.txt
+
+          mkdir -p trivy_input/docs
+          pip-compile -o trivy_input/docs/requirements.txt docs/requirements.txt
+
+          mkdir -p trivy_input/core
+          pip-compile -o trivy_input/core/requirements.txt requirements-core.txt
+
+          mkdir -p trivy_input/default
+          pip-compile -o trivy_input/core/requirements.txt requirements-default.txt
+
+          mkdir -p trivy_input/develop
+          pip-compile -o trivy_input/develop/requirements.txt requirements-dev.txt
+
+          mkdir -p trivy_input/base
+          pip-compile -o trivy_input/base/requirements.txt requirements.txt
+
+          mkdir -p trivy_input/tests
+          pip-compile -o trivy_input/tests/requirements.txt tests/requirements.txt
+
+      - name: Run Trivy Scan (vuln)
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
+        with:
+          scan-type: fs
+          format: "sarif"
+          scan-ref: trivy_input
+          scanners: vuln
+          output: trivy-results-vuln.sarif
+
+      - name: Upload SARIF file
+        if: ${{ always() }}
+        uses: github/codeql-action/upload-sarif@17a820bf2e43b47be2c72b39cc905417bc1ab6d0 # v3.28.8
+        with:
+          sarif_file: "trivy-results-vuln.sarif"
+
+      - name: Upload Trivy results artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: trivy-vuln-results
+          path: "${{ github.workspace }}/trivy-results-vuln.sarif"
+          retention-days: 7
+
+      - name: Upload deps list
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        if: always()
+        with:
+          name: python-deps-list
+          path: "${{ github.workspace }}/trivy_input"
+          retention-days: 7
+
+  Trivy-Scan-Misconfig:
+    runs-on: ubuntu-22.04
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Run Trivy Scan (dockerfile and secrets)
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
+        with:
+          scan-type: fs
+          format: "sarif"
+          scan-ref: .
+          scanners: misconfig,secret
+          output: trivy-results-misconfig.sarif
+
+      - name: Upload SARIF file
+        if: ${{ always() }}
+        uses: github/codeql-action/upload-sarif@17a820bf2e43b47be2c72b39cc905417bc1ab6d0 # v3.28.8
+        with:
+          sarif_file: "trivy-results-misconfig.sarif"
+
+      - name: Upload Trivy results artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: trivy-misconfig-results
+          path: "${{ github.workspace }}/trivy-results-misconfig.sarif"
+          retention-days: 7
+
   Bandit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    permissions:
+      security-events: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Python
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install bandit
+        run: pip install bandit[sarif]
+
       - name: Bandit Scanning
-        run: tox -e bandit-scan
-      - name: Upload Bandit artifact
-        uses: actions/upload-artifact@v4
+        run: bandit -r -c .ci/ipas_default.config . -f sarif -o bandit-results.sarif
+
+      - name: Upload SARIF file
+        if: ${{ always() }}
+        uses: github/codeql-action/upload-sarif@17a820bf2e43b47be2c72b39cc905417bc1ab6d0 # v3.28.8
         with:
-          name: bandit-report
-          path: .tox/bandit-report.txt
-        # Use always() to always run this step to publish scan results when there are test failures
+          sarif_file: bandit-results.sarif
+
+      - name: Upload Bandit artifact
         if: ${{ always() }}
-  call-notify-to-teams:
-    needs: [Trivy, Bandit]
-    if: |
-      always() &&
-      contains(needs.*.result, 'failure')
-    uses: ./.github/workflows/notify_teams.yml
-    secrets: inherit
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: bandit-results
+          path: bandit-results.sarif
+          retention-days: 7
+
+  CodeQL:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-22.04
+    permissions:
+      # required for all workflows
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["c-cpp", "python", "actions"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        with:
+          languages: ${{ matrix.language }}
+      - name: Setup Python
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: "3.10"
+
+      - name: Build
+        run: |
+          pip install build
+          python -m build
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        with:
+          category: "/language:${{matrix.language}}"
+
+      - name: Generate CodeQL Report
+        uses: rsdmike/github-security-report-action@a149b24539044c92786ec39af8ba38c93496495d # v3.0.4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          template: report
+          outputDir: codeql-${{ matrix.language }}
+
+      - name: Rename Report
+        shell: bash
+        continue-on-error: true
+        run: |
+          cd codeql-${{ matrix.language }}
+          mv "report.pdf" "codeql-${{ matrix.language }}.pdf"
+
+      - name: Upload Report
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: codeql-${{ matrix.language }}-results
+          path: codeql-${{ matrix.language }}/*.pdf
+          retention-days: 7
+
+  Summarize:
+    needs: [Trivy-Scan-Vuln, Trivy-Scan-Misconfig, Bandit, CodeQL]
+    if: always()
+    runs-on: ubuntu-22.04
+    steps:
+      # Create directory first
+      - name: Create results directory
+        run: mkdir -p all-results
+
+      # Download artifacts with error handling
+      - name: Download all results
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        continue-on-error: true # Don't fail if some tools didn't generate results
+        with:
+          pattern: "*-results"
+          merge-multiple: true
+          path: all-results
+
+      # Only upload if there are files
+      - name: Upload combined results
+        if: hashFiles('all-results/**/*') != ''
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: security-scan-results
+          path: all-results
+          retention-days: 7