Merge pull request #68 from yonhan3/umbrella-pid-feature

new spdx_deb script to create SPDX docs for DEBs built from src

README.md

@@ -49,6 +49,7 @@ Multiple Python scripts are developed to work together with these tools.
 - bomsh_index_ws.py script, which creates a blob index database for software build workspace.
 - bomsh_sbom.py script, which creates or updates SPDX SBOM documents with OmniBOR info.
 - bomsh_spdx_rpm.py script, which creates or updates SPDX SBOM documents for RPMs built from its src RPM.
+- bomsh_spdx_deb.py script, which creates or updates SPDX SBOM documents for DEBs built from its src.
 - bomsh_art_tree.py script, which grafts new subtrees or prunes existing subtrees of OmniBOR artifact trees.
 - bomsh_dynlib.py script, which creates raw_logfile of runtime-dependency fragments for ELF executables.
 - bomsh_pylib.py script, which creates raw_logfile of runtime-dependency fragments for Python scripts.
@@ -70,7 +71,7 @@ For a quick start of using the Bomsh tool, run the below command:
     $
     $ # the above should take only a few minutes, and the below may take tens of minutes
     $ wget https://buildinfos.debian.net/buildinfo-pool/s/sysstat/sysstat_11.7.3-1_all-amd64-source.buildinfo
-    $ bomsh/scripts/bomsh_rebuild_deb.py -f sysstat_11.7.3-1_all-amd64-source.buildinfo -d bomsh/scripts/sample_sysstat_cvedb.json -o outdir2 --syft_sbom --mmdebstrap_no_cleanup
+    $ bomsh/scripts/bomsh_rebuild_deb.py -f sysstat_11.7.3-1_all-amd64-source.buildinfo -d bomsh/scripts/sample_sysstat_cvedb.json -o outdir2 --syft_sbom --bomsh_spdx --mmdebstrap_no_cleanup
     $ grep -B1 -A3 CVElist outdir2/bomsher_out/bomsh_logfiles/bomsh_search_jsonfile-details.json
 
 Then explore and inspect all the output files in the outdir/bomsher_out directory,
@@ -82,7 +83,7 @@ contain the constructed OmniBOR tree with relevant metadata for the built RPM/DE
 the bomsh_logfiles/bomsh-index-* files contain the relevant package/blobs database,
 the syft_sbom/omnibor* files contain the syft-generated SPDX SBOM documents with ExternalRef OmniBOR identifier,
 and the bomsh_sbom/* files contain the SPDX SBOM documents with ExternalRef OmniBOR identifier
-generated by the bomsh_spdx_rpm.py script.
+generated by the bomsh_spdx_rpm.py or bomsh_spdx_deb.py script.
 
 Compile Bombash and Bomtrace from Source
 ----------------------------------------
@@ -138,7 +139,7 @@ Except for this difference, all other steps to generate OmniBOR documents are th
 
 By running with all C code instead of invoking Python scripts,
 Bomtrace3 saves a lot of process context switches overhead, thus improving the performance significantly over Bomtrace2.
-Bomtrace2 is a few (2x to 5x) times slower than the baseline run, while Bomtrace3 has only about 10% or 20% runtime overhead.
+Bomtrace2 is a few (2x to 5x) times slower than the baseline run, while Bomtrace3 has only about 20% runtime overhead.
 
 Generating OmniBOR Docs with Bomtrace2
 -------------------------------------

scripts/bomsh_rebuild_deb.py

@@ -200,7 +200,10 @@ def fix_broken_symlinks(bomsher_outdir):
     /tmp/bomsh_search_cve.py --derive_sbom -b omnibor_dir $cvedb_file_param -f $debfiles -vvv ; cp /tmp/bomsh_search_jsonfile* bomsh_logfiles/ ; \\
     # Extra handling of syft generated SPDX SBOM documents ; \\
     if [ "${SYFT_SBOM}" ]; then /tmp/bomsh_sbom.py -b omnibor_dir -F $debfiles -vv --output_dir syft_sbom --sbom_format spdx --force_insert ; fi ; \\
-    if [ "${SYFT_SBOM}" ]; then /tmp/bomsh_sbom.py -b omnibor_dir -F $debfiles -vv --output_dir syft_sbom --sbom_format spdx-json --force_insert ; fi ;
+    if [ "${SYFT_SBOM}" ]; then /tmp/bomsh_sbom.py -b omnibor_dir -F $debfiles -vv --output_dir syft_sbom --sbom_format spdx-json --force_insert ; fi ; \\
+    # Extra handling of bomsh-spdx generated SPDX SBOM documents ; \\
+    export PYTHONPATH=/root/tools-python/src:/root/beartype:/root/packageurl-python/src ; \\
+    if [ "${BOMSH_SPDX}" ]; then /tmp/bomsh_spdx_deb.py -F $debfiles --output_dir bomsh_sbom --sbom_server_url http://your.org ; fi ;
 '''
 
 def create_dockerfile(work_dir):
@@ -213,6 +216,14 @@ def create_dockerfile(work_dir):
     else:
         from_str = 'FROM debian:bookworm'
     dockerfile_str = from_str + g_bomsh_dockerfile_str
+    if args.bomsh_spdx:
+        # bomsh_spdx_deb.py requires additional python libraries
+        dockerfile_str = dockerfile_str.replace("rm -rf /var/lib/apt/lists/* ;",
+                "apt install python3-requests python3-license-expression python3-uritools python3-rdflib python3-xmltodict python3-yaml ; \\\n"
+                "    cd /root ; git clone https://github.com/spdx/tools-python.git ; \\\n"
+                "    git clone https://github.com/beartype/beartype.git ; \\\n"
+                "    git clone https://github.com/package-url/packageurl-python.git ; \\\n"
+                "    rm -rf /var/lib/apt/lists/* ;")
     dockerfile = os.path.join(work_dir, "Dockerfile")
     write_text_file(dockerfile, dockerfile_str)
 
@@ -251,6 +262,9 @@ def run_docker(buildinfo_file, output_dir):
     if args.syft_sbom:
         # Generate SBOM document with the syft tool
         docker_cmd += ' -e SYFT_SBOM=1'
+    if args.bomsh_spdx:
+        # Generate SPDX SBOM document with the bomsh_spdx_rpm.py tool
+        docker_cmd += ' -e BOMSH_SPDX=1'
     docker_cmd += ' -v ' + output_dir + ':/out $(docker build -t bomsher-deb -q ' + bomsher_indir + ')'
     verbose("==== Here is the docker run command: " + docker_cmd, LEVEL_1)
     os.system(docker_cmd)
@@ -289,6 +303,9 @@ def rtd_parse_options():
     parser.add_argument("--syft_sbom",
                     action = "store_true",
                     help = "run syft to generate DEB SBOM in spdx/spdx-json SBOM format")
+    parser.add_argument("--bomsh_spdx",
+                    action = "store_true",
+                    help = "run bomsh_spdx_deb.py to generate DEB SBOM in spdx/spdx-json SBOM format")
     parser.add_argument("--mmdebstrap_no_cleanup",
                     action = "store_true",
                     help = "do not cleanup chroot directory after mmdebstrap run")

scripts/bomsh_rebuild_rpm.py

@@ -141,9 +141,6 @@ def fix_broken_symlinks(bomsher_outdir):
     ./bootstrap && ./configure --enable-mpers=check && make ; \\
     cp src/strace /tmp/bomtrace2 ;
 
-# Set up SPDX tools-python environment
-RUN cd /root ; git clone https://github.com/spdx/tools-python.git ;
-
 # Bomtrace/Bomsh mock build run to generate OmniBOR documents
 # if BASELINE_REBUILD is not empty, then it will not use bomtrace2 to run mock, that is, the baseline run.
 # if CHROOT_CFG is not empty, then the provided mock chroot_cfg will be used, otherwise, default.cfg is used.
@@ -175,7 +172,7 @@ def fix_broken_symlinks(bomsher_outdir):
     if [ "${SYFT_SBOM}" ]; then /tmp/bomsh_sbom.py -b omnibor_dir -F $rpmfiles -vv --output_dir syft_sbom --sbom_format spdx-json ; fi ; \\
     # Extra handling of bomsh-spdx generated SPDX SBOM documents ; \\
     export PYTHONPATH=/root/tools-python/src ; \\
-    if [ "${BOMSH_SPDX}" ]; then /tmp/bomsh_spdx_rpm.py -r $rpmfiles --output_dir bomsh_sbom --sbom_server_url http://your.org ; fi ;
+    if [ "${BOMSH_SPDX}" ]; then /tmp/bomsh_spdx_rpm.py -F $rpmfiles --output_dir bomsh_sbom --sbom_server_url http://your.org ; fi ;
 '''
 
 def create_dockerfile(work_dir):
@@ -196,6 +193,7 @@ def create_dockerfile(work_dir):
         # bomsh_spdx_rpm.py requires additional python libraries from pip3
         bomsh_dockerfile_str = bomsh_dockerfile_str.replace("dnf clean all ;",
                 "pip3 install requests license-expression beartype uritools rdflib xmltodict pyyaml packageurl-python ; \\\n"
+                "    cd /root ; git clone https://github.com/spdx/tools-python.git ; \\\n"
                 "    dnf clean all ;")
     if args.bomsh_spdx and "almalinux:8" in from_str:
         # almalinux8 has python3.6 version as default, but we need at least python3.8 version for bomsh_spdx_rpm.py and spdx/tools-python library