[Security Policies] Generate templates - initial commit (#120)

* Add files via upload

* Update worker.js

* Merge pull request #1 from ofiriro3/editing-readme

Adding information to readme.md

* Merge pull request #2 from ofiriro3/editing-readme

Indentation + path for csp

* Update README.md

Adding a bit of information regarding the manual process of rules

* Merge pull request #3 from jeniawhite/add-json-gen

add json rule generator

* Add an option to generate report for missing rules (#4)

* Rule template schema changes - introducing `metadata` field

* temp solution for merge

* Initial commit

* move to subfolder

* revert temp

* mv .gitattributes

Co-authored-by: Evgeniy Belyi <jeniawhite92@gmail.com>
Co-authored-by: ofiriro3 <ofiriro3@Gmail.com>
Co-authored-by: Uri Weisman <68195305+uri-weisman@users.noreply.github.com>
Co-authored-by: Kfir Peled <61654899+kfirpeled@users.noreply.github.com>

cis-benchmark-pdf-data-serialization/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

cis-benchmark-pdf-data-serialization/.gitignore

@@ -0,0 +1,133 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+
+.idea

cis-benchmark-pdf-data-serialization/README.md

@@ -0,0 +1,57 @@
+# cis-benchmark-pdf-data-serialization
+
+The cis-benchmark serialization tool is an internal tool meant for creating the cis benchmark rule metadata.
+
+## Getting started 
+
+### Prerequisites
+- node.js version v17.6.0 and above
+
+### Running the project
+
+Install node dependencies `npm install`
+
+Go the `main.js` and edit the following:
+- Download the `CIS_Kubernetes_Benchmark_v1.6.0.pdf` file.
+- Set the `pdfPath` to your `CIS_Kubernetes_Benchmark_v1.6.0.pdf` file path.
+- **Optional** set the B_Page/E_page to the first/last rule page you want to convert.
+
+Project CLI: `node main.js --help`
+```
+Usage: cis-benchmark-pdf-serialization [options]
+
+CLI CIS Benchmark parser
+
+Options:
+  -V, --version              output the version number
+  -p, --pdfpath <string>
+  -r, --reportpath <string>
+  -h, --help                 display help for command
+  
+Example:
+  node main.js -p <BENCHMARK_PATH> -r <DIR_TO_STORE_REPORT>
+```
+
+### Guidelines on how to process generated rules
+
+There is no generation of markdown for the field values (remains a gap).
+This means that the metadata needs to be manually converted after generation.
+
+Here is the list of common things that needs to be addressed when manually editing the generated data:
+- In some cases there are problems with spaces when concating strings (heuristic needs to be improved).
+  Sometimes the name field strings are getting concated with a missing space between words.
+  Like this for example: 
+  Ensure that the --kubelet-client-certificate and --kubelet-client-keyarguments are set as appropriate (Automated).
+  Notice that `--kubelet-client-keyarguments` doesn't have a space and should be `--kubelet-client-key arguments`.
+  This can happen not only in names, but across other values as well.
+- profile_applicability field needs to be converted from `• Level 1 - Master Node` to `* Level 1 - Master Node`
+  or any other fields that have `•`, like bullets within the values should be converted to `*`.
+- Notice that there are several ways to assign values to a property in YAMLs (`|`, `>`).
+  `|` is mainly used for multiline values and is essential when you have codeblock inside the value.
+  `>` is mainly used for singleline values and it adds `\n` for every row end at the YAML, this option also supports inline code.
+- Code blocks must have newline/`\n` prior to every \`\`\` and newline/`\n` after every \`\`\`.
+- Some rules might have bold values inside the value of the rule that will need to be converted with `**VALUE**`.
+  Example of such case are rules 1.2.15, 2.4, 2.5, 2.6, notice that they have `**Note**`.
+- Some rules do not have all of the properties and if that's the case then you will need to add empty properties, because ignoring that will cause the tests to fail.
+  One of such cases is rule 5.1.3 that doesn't have properties `default_value`, `references`.
+- There are also instances of EKS rules having `Audit1`, `Audit2` properties which are being parsed but not picked by the script.

cis-benchmark-pdf-data-serialization/gen-json-rules.js

@@ -0,0 +1,58 @@
+"use strict";
+
+const _ = require("lodash");
+const fs = require("fs");
+const YAML = require("yaml");
+const path = require("path")
+
+const CSP_PATH = "../csp-security-policies";
+
+(async function () {
+  const rules = fs.readdirSync(
+    path.join(CSP_PATH, "/compliance/cis_k8s/rules/")
+  );
+  rules.forEach((rule) => {
+    const yaml_doc = new YAML.Document();
+    const exist_raw_rule = fs.readFileSync(
+      path.join(CSP_PATH, `compliance/cis_k8s/rules/${rule}/data.yaml`),
+      "utf-8"
+    );
+    const rule_obj = YAML.parse(exist_raw_rule).metadata;
+    rule_obj.enabled = true;
+    rule_obj.muted = false;
+    rule_obj.rego_rule_id = rule;
+    const integration_rule = migrateCspRuleMetadata({id: rule_obj.id, type: 'csp-rule-template', attributes: rule_obj});
+
+    fs.writeFileSync(
+      path.join(`../integrations/packages/cloud_security_posture/kibana/csp_rule_template/`,`${integration_rule.id}.json`),
+      JSON.stringify(integration_rule, null, 4),
+      "utf-8"
+    );
+  });
+})();
+
+/**
+ * Migrating csp rule template schema from version 8.3.0 to 8.4.0
+ * Main changes:
+ * - introducing `metadata` field
+ */
+function migrateCspRuleMetadata(doc) {
+  const { enabled, muted, ...metadata } = doc.attributes;
+  return {
+    ...doc,
+    attributes: {
+      enabled,
+      muted,
+      metadata: {
+        ...metadata,
+        impact: metadata.impact || undefined,
+        default_value: metadata.default_value || undefined,
+        references: metadata.references || undefined,
+      },
+    },
+    migrationVersion: {
+      "csp-rule-template": "8.4.0"
+    },
+    coreMigrationVersion: "8.4.0"
+  };
+}

cis-benchmark-pdf-data-serialization/main.js

@@ -0,0 +1,92 @@
+"use strict";
+
+const pdfjsLib = require("pdfjs-dist/legacy/build/pdf.js");
+const Worker = require("./worker.js");
+const _ = require("lodash");
+const YAML = require("yaml");
+const fs = require("fs");
+const path = require("path")
+const papa = require("papaparse");
+const { Command } = require('commander');
+
+const cspPath = "../csp-security-policies"
+const BENCHMARK = "Kubernetes"; //"Kubernetes"; // EKS
+//   "./CIS_Amazon_Elastic_Kubernetes_Service_(EKS)_Benchmark_v1.0.1.pdf"; //"./CIS_Kubernetes_Benchmark_v1.6.0.pdf";
+const w = new Worker({benchmark: BENCHMARK});
+const program = new Command();
+
+program
+  .name('cis-benchmark-pdf-serialization')
+  .description('CLI CIS Benchmark parser')
+  .version('0.0.1');
+
+program
+  .option('-p, --pdfpath <string>')
+  .option('-r, --reportpath <string>')
+
+program.parse();
+const options = program.opts();
+const pdfPath = options.pdfpath;
+const reportPath = options.reportpath;
+
+try {
+    (async function () {
+        const missingRules = [];
+        const loadingTask = pdfjsLib.getDocument(pdfPath);
+        const doc = await loadingTask.promise;
+        const numPages = doc.numPages;
+        console.log("# Document Loaded");
+        console.log("Number of Pages: " + numPages);
+        const B_PAGE = 17; //13; // 254; // 260; //17;
+        const E_PAGE = 262; // 132; //255; // 261; //18;
+        for (let pageNum = B_PAGE; pageNum <= E_PAGE; pageNum++) {
+            const page = await doc.getPage(pageNum);
+            const content = await page.getTextContent();
+            // console.log("# Page " + pageNum);
+            w.addItems(content.items, pageNum);
+            page.cleanup();
+        }
+        const ruleObjs = w.Run();
+        // if (fs.existsSync("./yamls")) fs.rmSync("./yamls", { recursive: true });
+        // fs.mkdirSync("./yamls");
+        ruleObjs.forEach((ruleObj) => {
+            // console.log(ruleObj);
+            const r_number = ruleObj.metadata.tags[2]
+                .split(" ")[1]
+                .replaceAll(".", "_");
+            // ruleObj = _.omit(ruleObj, "metadata.tags");
+            const yaml_doc = new YAML.Document();
+            // console.log(doc.toString());
+            if (isRuleImplemented(r_number)) {
+                if (!fs.existsSync(path.join(cspPath, `compliance/cis_k8s/rules/cis_${r_number}/data.yaml`))) {
+                    console.log("JENIA NEW RULE", r_number);
+                    yaml_doc.contents = ruleObj;
+                    // console.log("content")
+                    // console.log(yaml_doc.contents)
+                    fs.writeFileSync(path.join(cspPath, `compliance/cis_k8s/rules/cis_${r_number}/data.yaml`), yaml_doc.toString());
+                    console.log("after write")
+                }
+            } else {
+                missingRules.push({number: r_number, name: ruleObj.metadata.name})
+            }
+        });
+
+        if (reportPath) {
+            const filename = "missing_rules.csv"
+            const columns = ["number", "name"]
+            let csv = papa.unparse({ data: missingRules, fields: columns});
+            if (csv == null) return;
+
+            fs.writeFile(path.join(reportPath, filename), csv, (err) => {
+                if (err) throw err;
+                console.log('The file has been saved!');
+            });
+        }
+    })();
+} catch (error) {
+    console.error(err);
+}
+
+function isRuleImplemented(r_number) {
+    return fs.existsSync(path.join(cspPath, `/compliance/cis_k8s/rules/cis_${r_number}`))
+}