From 4d894f031115e7f7bc8492b2200d7faca1e7fa1e Mon Sep 17 00:00:00 2001 From: Yongkui Han Date: Fri, 16 Feb 2024 00:41:29 +0000 Subject: [PATCH 1/2] add cpe23Type ExternalRef and more SPDX output formats Signed-off-by: Yongkui Han --- scripts/bomsh_spdx_deb.py | 47 ++++++++++++++++++++++++++++++++----- scripts/bomsh_spdx_rpm.py | 49 ++++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/scripts/bomsh_spdx_deb.py b/scripts/bomsh_spdx_deb.py index 5c62c26..4e2c082 100755 --- a/scripts/bomsh_spdx_deb.py +++ b/scripts/bomsh_spdx_deb.py @@ -220,6 +220,12 @@ def get_pkg_name(pkg_data): pkg_name = pkg_data['Source'] + ".Source" return pkg_name +def get_pkg_version(pkg_data): + pkg_ver = 'UNKNOWN' + if 'Version' in pkg_data: + pkg_ver = pkg_data['Version'] + return pkg_ver + def deb_pkg_nvra(pkg_data): # "Name : gcc" #pkg_name = pkg_data['Name'] @@ -327,7 +333,6 @@ def build_pkg_purl(rel_data, pkg_data): #version=f"{pkg_data['Version']}-{pkg_data['Release']}", qualifiers=qual_d, subpath=None) - return str(purl) def make_purl_ref(pkg_data, os_rel_data): @@ -339,7 +344,28 @@ def make_purl_ref(pkg_data, os_rel_data): locator=purl, # comment="external reference comment", ) + return ref +def build_pkg_cpe(rel_data, pkg_data): + + # cpe:::::::::::: + # like "cpe:2.3:o:canonical:ubuntu_linux:10.04:-:lts:*:*:*:*:*" + # or "cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:" + + pkg_name = get_pkg_name(pkg_data) + pkg_ver = get_pkg_version(pkg_data) + wfn = ["cpe", "2.3", "a", rel_data['ID'], pkg_name, pkg_ver, "*", "*", "*", "*", "*", "*", "*"] + return ":".join(wfn) + +def make_cpe_ref(pkg_data, os_rel_data): + cpe = build_pkg_cpe(os_rel_data, pkg_data) + + ref = ExternalPackageRef( + category=ExternalPackageRefCategory.SECURITY, + reference_type="cpe23Type", + locator=cpe, + # comment="external reference comment", + ) return ref def build_base_spdx(pkg_name, doc_uuid): @@ -401,7 +427,8 @@ def spdx_add_package(spdx_doc, rpm_file, bom_id, file_verification_code, os_rel_ locator=f'gitoid:blob:sha1:{bom_id}', comment=f'{ADG_SERVER_URL}/adg/tree/{bom_id}', ), - make_purl_ref(pkg_data, os_rel_data) + make_purl_ref(pkg_data, os_rel_data), + make_cpe_ref(pkg_data, os_rel_data) ] # license_concluded=spdx_licensing.parse("GPL-2.0-only OR MIT"), # license_info_from_files=[spdx_licensing.parse("GPL-2.0-only"), spdx_licensing.parse("MIT")], @@ -760,10 +787,16 @@ def handle_files(rel_data): # The document namespace will be something like this: # https://sbom.your-org.com/spdxdocs/sysstat-11.7.3-9.el8.src.rpm-b184657e-6b09-48d5-a5fc-df2f106f40b5 # so the path will be: sysstat-11.7.3-9.el8.src.rpm-b184657e-6b09-48d5-a5fc-df2f106f40b5.spdx.json - output_fn = f'{os.path.basename(urlsplit(spdx_doc.creation_info.document_namespace).path)}.spdx.json' - output_file = os.path.join(output_dir, output_fn) - write_file(spdx_doc, output_file) - omnibor_sbom_docs.append(output_file) + doc_basename = os.path.basename(urlsplit(spdx_doc.creation_info.document_namespace).path) + if args.spdx_format: + format_list = args.spdx_format.split(",") + else: + format_list = ["spdx", "spdx.json", "spdx.rdf", "spdx.xml", "spdx.yaml"] + for suffix in format_list: + output_fn = f'{doc_basename}.{suffix}' + output_file = os.path.join(output_dir, output_fn) + write_file(spdx_doc, output_file) + omnibor_sbom_docs.append(output_file) print("\nDone. All bomsh created SPDX SBOM documents with OmniBOR info are: " + str(omnibor_sbom_docs)) @@ -794,6 +827,8 @@ def rtd_parse_options(): help = "the directory with bomsh log files") parser.add_argument("--debs_dir", help = "the directory with Debian package files") + parser.add_argument('--spdx_format', + help = "comma-separated list of output SPDX SBOM format, like spdx,json,rdf,xml,yaml, etc.") parser.add_argument("--keep_intermediate_files", action = "store_true", help = "after run completes, keep all intermediate files like unbundled packages, etc.") diff --git a/scripts/bomsh_spdx_rpm.py b/scripts/bomsh_spdx_rpm.py index 019684f..3184bd2 100755 --- a/scripts/bomsh_spdx_rpm.py +++ b/scripts/bomsh_spdx_rpm.py @@ -191,6 +191,14 @@ def rpm_query_pkg(rpm): pkg_data = parse_pkg_info(pkg_info) return pkg_data +def get_pkg_name(pkg_data): + pkg_name = '' + if 'Name' in pkg_data: + pkg_name = pkg_data['Name'] + else: + pkg_name = pkg_data['Source'] + ".Source" + return pkg_name + def rpm_pkg_nvra(pkg_data): # "Name : gcc" pkg_name = pkg_data['Name'] @@ -288,7 +296,6 @@ def build_pkg_purl(rel_data, pkg_data): version=f"{pkg_data['Version']}-{pkg_data['Release']}", qualifiers=qual_d, subpath=None) - return str(purl) def make_purl_ref(pkg_data, os_rel_data): @@ -300,7 +307,28 @@ def make_purl_ref(pkg_data, os_rel_data): locator=purl, # comment="external reference comment", ) + return ref + +def build_pkg_cpe(rel_data, pkg_data): + + # cpe:::::::::::: + # like "cpe:2.3:o:canonical:ubuntu_linux:10.04:-:lts:*:*:*:*:*" + # or "cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:" + pkg_name = get_pkg_name(pkg_data) + pkg_ver = f"{pkg_data['Version']}-{pkg_data['Release']}" + wfn = ["cpe", "2.3", "a", rel_data['ID'], pkg_name, pkg_ver, "*", "*", "*", "*", "*", "*", "*"] + return ":".join(wfn) + +def make_cpe_ref(pkg_data, os_rel_data): + cpe = build_pkg_cpe(os_rel_data, pkg_data) + + ref = ExternalPackageRef( + category=ExternalPackageRefCategory.SECURITY, + reference_type="cpe23Type", + locator=cpe, + # comment="external reference comment", + ) return ref def build_base_spdx(pkg_name, doc_uuid): @@ -360,7 +388,8 @@ def spdx_add_package(spdx_doc, rpm_file, bom_id, file_verification_code, os_rel_ locator=f'gitoid:blob:sha1:{bom_id}', comment=f'{ADG_SERVER_URL}/adg/tree/{bom_id}', ), - make_purl_ref(pkg_data, os_rel_data) + make_purl_ref(pkg_data, os_rel_data), + make_cpe_ref(pkg_data, os_rel_data) ] # license_concluded=spdx_licensing.parse("GPL-2.0-only OR MIT"), # license_info_from_files=[spdx_licensing.parse("GPL-2.0-only"), spdx_licensing.parse("MIT")], @@ -712,10 +741,16 @@ def handle_files(rel_data): # The document namespace will be something like this: # https://sbom.your-org.com/spdxdocs/sysstat-11.7.3-9.el8.src.rpm-b184657e-6b09-48d5-a5fc-df2f106f40b5 # so the path will be: sysstat-11.7.3-9.el8.src.rpm-b184657e-6b09-48d5-a5fc-df2f106f40b5.spdx.json - output_fn = f'{os.path.basename(urlsplit(spdx_doc.creation_info.document_namespace).path)}.spdx.json' - output_file = os.path.join(output_dir, output_fn) - write_file(spdx_doc, output_file) - omnibor_sbom_docs.append(output_file) + doc_basename = os.path.basename(urlsplit(spdx_doc.creation_info.document_namespace).path) + if args.spdx_format: + format_list = args.spdx_format.split(",") + else: + format_list = ["spdx", "spdx.json", "spdx.rdf", "spdx.xml", "spdx.yaml"] + for suffix in format_list: + output_fn = f'{doc_basename}.{suffix}' + output_file = os.path.join(output_dir, output_fn) + write_file(spdx_doc, output_file) + omnibor_sbom_docs.append(output_file) print("\nDone. All bomsh created SPDX SBOM documents with OmniBOR info are: " + str(omnibor_sbom_docs)) @@ -746,6 +781,8 @@ def rtd_parse_options(): help = "the directory with bomsh log files") parser.add_argument("--rpms_dir", help = "the directory with RPM files") + parser.add_argument('--spdx_format', + help = "comma-separated list of output SPDX SBOM format, like spdx,json,rdf,xml,yaml, etc.") parser.add_argument("--keep_intermediate_files", action = "store_true", help = "after run completes, keep all intermediate files like unbundled packages, etc.") From deb8b7c41e930e3984e215776099ca3dde9be579 Mon Sep 17 00:00:00 2001 From: Yongkui Han Date: Fri, 16 Feb 2024 00:43:05 +0000 Subject: [PATCH 2/2] bomtrace3 handle OpenWRT piggy.gzip incbin for kernel build Signed-off-by: Yongkui Han --- .devcontainer/src/bomsh_hook.c | 67 ++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/.devcontainer/src/bomsh_hook.c b/.devcontainer/src/bomsh_hook.c index 9481043..db9c41e 100644 --- a/.devcontainer/src/bomsh_hook.c +++ b/.devcontainer/src/bomsh_hook.c @@ -2032,6 +2032,41 @@ static void bomsh_try_get_ld_subfiles(bomsh_cmd_data_t *cmd) bomsh_get_gcc_subfiles(cmd, (char **)linker_skip_tokens, num_skip_tokens); } +// This is for OpenWRT kernel build, and the below should be the piggy_gzip_cmd_file content: +// echo 'cmd_arch/arm/boot/compressed/piggy.gzip := (cat arch/arm/boot/compressed/../Image | gzip -n -f -9 > arch/arm/boot/compressed/piggy.gzip) || (rm -f arch/arm/boot/compressed/piggy.gzip ; false)' > arch/arm/boot/compressed/.piggy.gzip.cmd" + +// pigg_gzip_file is absolute path with /root/pwd/path/piggy.gzip format +static char *handle_piggy_gzip(char *piggy_gzip_file) +{ + char buf[PATH_MAX]; + bomsh_log_printf(6, "incbin piggy_gzip file: %s\n", piggy_gzip_file); + char *end = stpcpy(buf, piggy_gzip_file); + strcpy(end - strlen("piggy.gzip"), ".piggy.gzip.cmd"); + if (!is_regular_file(buf)) { + return NULL; + } + bomsh_log_printf(6, "found piggy_gzip_cmd file: %s the whole gzip_cmd_str:\n", buf); + char *content = bomsh_read_file(buf, NULL); + bomsh_log_string(6, content); + + char *real_image = NULL; + char delim[] = " "; + char *ptr = strtok(content, delim); + while(ptr != NULL) + { + if (strcmp(ptr, "(cat") == 0) { + // the next token is the real image + ptr = strtok(NULL, delim); + real_image = strdup(ptr); + bomsh_log_printf(6, "for piggy_gzip file, found real image: %s\n", real_image); + break; + } + ptr = strtok(NULL, delim); + } + free(content); + return real_image; +} + /* * Read the piggy.S file and return the included binary file vmlinux.bin * @@ -2059,14 +2094,24 @@ z_output_len = 43225848 .incbin "arch/arm/boot/compressed/piggy_data" .globl input_data_end input_data_end: +[root@87e96394b5b5 linux]# +[root@rtp base]# cat kernel/linux-mvl-3.14/arch/arm/boot/compressed/piggy.gzip.S + .section .piggydata,#alloc + .globl input_data +input_data: + .incbin "arch/arm/boot/compressed/piggy.gzip" + .globl input_data_end +input_data_end: +[root@rtp base]# * * Note, it can be either space character or tab character after ".incbin" */ static char * bomsh_read_piggy_S_file(bomsh_cmd_data_t *cmd, char *piggy_S_file) { char buf[PATH_MAX]; - char *afile = get_real_path2(cmd, piggy_S_file, buf); - char *content = bomsh_read_file(afile, NULL); + char *piggy_S_path = get_real_path2(cmd, piggy_S_file, buf); + //bomsh_log_printf(6, "piggy_S file abspath: %s\n", piggy_S_path); + char *content = bomsh_read_file(piggy_S_path, NULL); char *p = content; char *inc_bin_str = NULL; while (*p) { @@ -2081,8 +2126,16 @@ static char * bomsh_read_piggy_S_file(bomsh_cmd_data_t *cmd, char *piggy_S_file) // remove the .gz to get the real vmlinux_bin file *(end - 3) = 0; } - inc_bin_str = strdup(start + 1); - bomsh_log_printf(8, "\nFound vmlinux_bin: %s from piggy_S file: %s\n", inc_bin_str, piggy_S_file); + if (strcmp(bomsh_basename(start + 1), "piggy.gzip") == 0) { + //bomsh_log_string(6, "found piggy.gzip incbin file, will read .piggy.gzip.cmd file for real image.\n"); + // start+1 points to "arch/arm/boot/compressed/piggy.gzip" string + char *piggy_gzip_file = get_real_path2(cmd, start+1, buf); + inc_bin_str = handle_piggy_gzip(piggy_gzip_file); + } + if (!inc_bin_str) { + inc_bin_str = strdup(start + 1); + } + bomsh_log_printf(8, "\nFound real image vmlinux_bin: %s from piggy_S file: %s\n", inc_bin_str, piggy_S_file); break; } } @@ -2098,16 +2151,14 @@ static char * find_piggy_S_file(char *output_file, char **input_files) return NULL; } char *outfile = bomsh_basename(output_file); - //if (strcmp(outfile, "piggy.o") && strcmp(outfile, "piggy.gzip.o")) { - if (strcmp(outfile, "piggy.o")) { + if (strcmp(outfile, "piggy.o") && strcmp(outfile, "piggy.gzip.o")) { return NULL; } char **p = input_files; while (*p) { char *token = *p; p++; char *name = bomsh_basename(token); - //if (strcmp(name, "piggy.S") == 0 || strcmp(name, "piggy.gzip.S") == 0) { - if (strcmp(name, "piggy.S") == 0) { + if (strcmp(name, "piggy.S") == 0 || strcmp(name, "piggy.gzip.S") == 0) { return token; } }