[Security Policies] Remove broken refs (#119)

* works single threaded

* add concurrency

* use await

cis_policies_generator/config/default.json

@@ -1,7 +1,8 @@
 {
     "version": 0.1,
     "output_folder": "output",
-    "output_filename": "combined.json",
+    "output_filename": "combined.yaml",
     "benchmarks_folder": "input",
-    "uuid_seed": "5d8d0dd5-acd2-4c46-b565-aa1fb03617af"
+    "uuid_seed": "5d8d0dd5-acd2-4c46-b565-aa1fb03617af",
+    "check_for_broken_references": true
 }

cis_policies_generator/src/fixBrokenReferences.ts

@@ -0,0 +1,59 @@
+import config from "config";
+import axios, {AxiosError} from "axios";
+
+const check: string = config.get("check_for_broken_references");
+
+const cache: HttpCache = {};
+
+function setCache(key: string, code: number) {
+    cache[key] = code;
+}
+
+function getCache(key: string): number {
+    return cache[key];
+}
+
+function logResponse(link: string, code: number, isFromCache: boolean) {
+    const term = isFromCache ? "via cache" : "";
+    console.log("Got", code, term, "for", link);
+}
+
+async function checkReference(link: string): Promise<boolean> {
+    const code = getCache(link);
+    if (!isNaN(code)) {
+        logResponse(link, code, true);
+        return code == 200;
+    }
+    try {
+        const res = await axios.head(link)
+        logResponse(link, res.status, false);
+        setCache(link, res.status);
+        return res.status == 200;
+    } catch (err) {
+        if (err instanceof AxiosError && err.response) {
+            setCache(link, err.response.status);
+            logResponse(link, err.response.status, false);
+        } else {
+            // If we got here, it means that we failed to reach the server because of things that are IN OUR CONTROL
+            // (e.g. timeout, socket reset)
+            console.log(err);
+            process.abort();
+        }
+        return false;
+    }
+}
+
+async function removeIfBroken(references: string[]) {
+    for (let i = references.length - 1; i >= 0; i--)
+        if (!(await checkReference(references[i])))
+            references.splice(i, 1)
+}
+
+export async function FixBrokenReferences(parsed_benchmarks: BenchmarkSchema[]): Promise<BenchmarkSchema[]> {
+    if (!check)
+        return parsed_benchmarks;
+    await Promise.all(parsed_benchmarks.map(
+        async bench => await Promise.all(bench.rules.map(
+            async rule => await removeIfBroken(rule.references)))))
+    return parsed_benchmarks
+}

cis_policies_generator/src/index.ts

@@ -4,6 +4,7 @@ import config from 'config';
 import xlsx from 'node-xlsx';
 import path from 'path';
 import YAML from 'yaml';
+import {FixBrokenReferences} from "./fixBrokenReferences";
 
 const {v5: uuid} = require('uuid');
 
@@ -146,13 +147,14 @@ function generateOutputFiles(benchmarks: BenchmarkSchema[]): void {
     fs.writeFileSync(output_folder + "/" + config.get("output_filename"), YAML.stringify(result));
 }
 
-function main(): void {
+
+async function main(): Promise<void> {
     // Make sure output folder exists an is empty
     generateOutputFolder();
-
     const parsed_benchmarks = parseBenchmarks(benchmarks_folder)
+    await FixBrokenReferences(parsed_benchmarks)
     generateOutputFiles(parsed_benchmarks);
     console.log("Done!");
 }
 
-main()
+main().then(r => r)

cis_policies_generator/src/types.d.ts

@@ -45,6 +45,10 @@ interface SpreadsheetTab {
     data: string[][];
 }
 
+interface HttpCache {
+    [key: string]: number
+}
+
 interface RuleSchema {
     audit: string;
     rule_number: string;

cis_policies_generator/tsconfig.json

@@ -1,15 +1,14 @@
 {
   "compilerOptions": { /* Docs: https://www.typescriptlang.org/tsconfig */
     "target": "ES2020",
-    "module": "commonjs",
     "outDir": "bin",
     "rootDir": "src",
     "strict": true,
     "noImplicitAny": true,
     "moduleResolution": "node",
     "lib": [
       "es6",
-      "es2021"
+      "es2022"
     ],
     "esModuleInterop": true,
     "skipLibCheck": true,