DOCSP-40534 Script to scrape licensing info from OM SBOM (#485)

Author: pierwill

.gitignore

@@ -24,3 +24,4 @@ source/figures/
 giza.log
 backups/*
 .vscode/
+tools/sbom-tool/sbom.json

tools/sbom-tool/Cargo.lock

@@ -0,0 +1,11 @@
+[package]
+name = "license-tool"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+iter_tools = "0.18.0"
+serde = { version = "1.0.203", features = ["derive"]}
+serde_json = "1.0.117"

tools/sbom-tool/Cargo.toml

@@ -0,0 +1,24 @@
+# Requirements
+
+Install [Rust](https://www.rust-lang.org/learn/get-started).
+
+# Instructions
+
+1. Download the SBOM (software bill of materials) for Ops Manager from here: https://parsley.mongodb.com/taskFile/mms_ops_manager_main_ssdlc_PACKAGE_SILK_SBOM_21ef545d4cdcde67d2eaa7f709e62a080db80cc6_24_05_30_14_41_10/0/Augmented%20SBOM?bookmarks=0,50069
+   (This location may change in the future. Ask engineering for latest if you have questions.)  
+   - Click *Details* -> *Raw* to downlaod the raw JSON. Name the file `sbom.json`.
+2. Place `sbom.json` in `docs-ops-manager/tools/sbom-tool`.
+3. Change to the tool directory: `cd docs-ops-manager/tools/sbom-tool/`.
+4. Run `cargo run`
+5. Copy to the output file (`out.rst`) to the appropriate section of `docs-ops-manager/source/reference/third-party-licenses.txt`.
+
+# Troubleshooting
+
+If you run into an error with `cargo run`, 
+try adding the following text to `~/.carg/config`
+(create this file if it does not exist):
+
+```
+[net]
+git-fetch-with-cli = true
+```

tools/sbom-tool/README.md

@@ -0,0 +1,32 @@
+.. list-table::
+   :widths: 50 50
+   :header-rows: 1
+   :class: licenses
+
+   * - Package
+     - Copyright
+
+   * - `pikaday <https://www.npmjs.com/package/pikaday/v/1.8.2>`__
+     - Copyright © 2013-2014 David Bushell
+     
+   * - `tslib <https://www.npmjs.com/package/tslib/v/2.6.2>`__
+     - Copyright © Microsoft Corporation
+
+   * - `github.com/davecgh/go-spew <https://pkg.go.dev/github.com/davecgh/go-spew@v1.1.2-0.20180830191138-d8f796af33cc>`__
+     - 
+
+   * - `github.com/decred/dcrd/crypto/blake256 <https://pkg.go.dev/github.com/decred/dcrd/crypto/blake256@v1.0.1>`__
+     - 
+
+   * - `github.com/decred/dcrd/dcrec/secp256k1 <https://pkg.go.dev/github.com/decred/dcrd/dcrec/secp256k1@v4.2.0>`__
+     - 
+
+   * - `github.com/gogo/protobuf <https://pkg.go.dev/github.com/gogo/protobuf@v1.3.2>`__
+     - 
+
+   * - `github.com/davecgh/go-spew <https://pkg.go.dev/github.com/davecgh/go-spew@v1.1.1>`__
+     - 
+
+   * - `github.com/howeyc/gopass <https://pkg.go.dev/github.com/howeyc/gopass@v0.0.0-20170109162249-bf9dde6d0d2c>`__
+     - 
+     

tools/sbom-tool/example.rst

@@ -0,0 +1,8 @@
+pub const TABLE_HEADER: &str = r#".. list-table::
+   :widths: 20 20 60
+   :header-rows: 1
+   :class: licenses"#;
+
+pub const COLUMN_HEADERS: &str = r#"   * - Package
+     - License
+     - Copyright"#;

tools/sbom-tool/src/constants.rs

@@ -0,0 +1,58 @@
+use std::fmt;
+
+use crate::sbom::Component;
+use iter_tools::Itertools;
+
+// Example:
+//
+// * - `pikaday <https://www.npmjs.com/package/pikaday/v/1.8.2>`__
+//   - Copyright © 2013-2014 David Bushell
+//
+impl fmt::Display for Component {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let name = self.name.clone().unwrap();
+
+        // Get a URL for this component.
+        // Prefer a github repo URL, but otherwise use any given URL.
+        let mut url = String::new();
+        let refs = self.clone().external_references;
+        if refs.is_some() {
+            // println!("{:#?}", refs.clone().unwrap());
+
+            for r in refs.clone().unwrap() {
+                if r.url.contains("https://github") {
+                    url = r.url.replace("git+", "").replace(".git", "");
+                } else {
+                    url = r.url;
+                }
+            }
+        }
+
+        // Licenses
+        let mut licenses: Vec<String> = vec![];
+        let ls = self.clone().licenses;
+        if ls.is_some() {
+            for l in ls.unwrap() {
+                let license = l.to_string();
+                if license != "Other" && license != "NOASSERTION" {
+                    licenses.push(license);
+                }
+            }
+        }
+
+        // Format copyright text into one line
+        let mut copyright = self.copyright.clone().unwrap_or(String::from(""));
+        copyright = copyright.replace('\n', "\n       ");
+
+        // Here is where we actually create the string that will go into
+        // the RST file, using the data from above.
+        write!(
+            f,
+            "   * - `{} <{}>`__\n     - {}\n     - {}\n\n",
+            name,
+            url,
+            licenses.iter().format(", "),
+            copyright,
+        )
+    }
+}

tools/sbom-tool/src/fmt.rs

@@ -0,0 +1,42 @@
+use std::fs::read_to_string;
+
+use sbom::Component;
+
+use crate::constants::{COLUMN_HEADERS, TABLE_HEADER};
+use crate::sbom::Sbom;
+
+mod constants;
+mod fmt;
+mod sbom;
+
+fn main() {
+    let file_path = "sbom.json".to_owned();
+    let contents = read_to_string(file_path).expect("Couldn't find or load that file.");
+
+    let sbom: Sbom = serde_json::from_str(&contents).expect("Error unmarshalling JSON");
+
+    let mut output = String::new();
+    output += TABLE_HEADER;
+    output += "\n\n";
+    output += COLUMN_HEADERS;
+    output += "\n\n";
+
+    let mut components: Vec<Component> = sbom.components;
+    components.sort_by(|a, b| {
+        a.clone()
+            .name
+            .unwrap()
+            .to_lowercase()
+            .cmp(&b.clone().name.unwrap().to_lowercase())
+    });
+    components.sort_by_key(|c| c.clone().licenses);
+
+    for comp in &components {
+        if comp.licenses.is_some() {
+            output += &format!("{}", comp);
+        }
+    }
+
+    // println!("{}", output);
+    std::fs::write("out.rst", output).expect("Error writing output file.");
+}

tools/sbom-tool/src/main.rs

@@ -0,0 +1,57 @@
+use std::collections::BTreeMap;
+
+use serde::Deserialize;
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Sbom {
+    pub components: Vec<Component>,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Component {
+    pub name: Option<String>,
+    pub copyright: Option<String>,
+    #[serde(rename = "externalReferences")]
+    pub external_references: Option<Vec<ExternalReference>>,
+    pub licenses: Option<Vec<License>>,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[serde(transparent)]
+pub struct License {
+    pub license_inner: BTreeMap<String, BTreeMap<String, String>>,
+}
+
+impl std::fmt::Display for License {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let out = self.license_inner.first_key_value().unwrap();
+        let (_, out) = out;
+        let out = out.first_key_value().unwrap();
+
+        write!(f, "{}", out.1)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ExternalReference {
+    #[serde(rename = "type")]
+    pub _type: ExternalReferenceType,
+    pub url: String,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename = "type")]
+pub enum ExternalReferenceType {
+    #[serde(rename = "distribution")]
+    Distribution,
+    #[serde(rename = "distribution-intake")]
+    DistributionIntake,
+    #[serde(rename = "evidence")]
+    Evidence,
+    #[serde(rename = "other")]
+    Other,
+    #[serde(rename = "vcs")]
+    Vcs,
+    #[serde(rename = "website")]
+    Website,
+}