Merge remote-tracking branch 'upstream/master'

gchq · May 9, 2019 · f9354c8 · f9354c8
2 parents 328c0ad + 01f0625
commit f9354c8
Show file tree

Hide file tree

Showing 48 changed files with 5,465 additions and 3,308 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -1 +1,2 @@
 src/core/vendor/**
+src/web/static/clippy_assets/**
diff --git a/.travis.yml b/.travis.yml
@@ -30,8 +30,9 @@ deploy:
     skip_cleanup: true
     api_key:
       secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
+    file_glob: true
     file:
-      - build/prod/cyberchef.htm
+      - build/prod/*.zip
       - build/node/CyberChef.js
     on:
       repo: gchq/CyberChef

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,18 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.31.0] - 2019-04-12
+- The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
+
+### [8.30.0] - 2019-04-12
+- 'Decode Protobuf' operation added [@n1474335] | [#533]
+
+### [8.29.0] - 2019-03-31
+- 'BLAKE2s' and 'BLAKE2b' hashing operations added [@h345983745] | [#525]
+
+### [8.28.0] - 2019-03-31
+- 'Heatmap Chart', 'Hex Density Chart', 'Scatter Chart' and 'Series Chart' operation added [@artemisbot] [@tlwr] | [#496] [#143]
+
 ### [8.27.0] - 2019-03-14
 - 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516]
 - See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations.
@@ -118,6 +130,10 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
+[8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
+[8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
+[8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
 [8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0
 [8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0
 [8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0
@@ -159,6 +175,7 @@ All major and minor version changes will be documented in this file. Details of
 [@h345983745]: https://github.com/h345983745
 [@s2224834]: https://github.com/s2224834
 [@artemisbot]: https://github.com/artemisbot
+[@tlwr]: https://github.com/tlwr
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
 [@JustAnotherMark]: https://github.com/JustAnotherMark
@@ -175,6 +192,7 @@ All major and minor version changes will be documented in this file. Details of
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
+[#143]: https://github.com/gchq/CyberChef/pull/143
 [#224]: https://github.com/gchq/CyberChef/pull/224
 [#239]: https://github.com/gchq/CyberChef/pull/239
 [#248]: https://github.com/gchq/CyberChef/pull/248
@@ -209,5 +227,8 @@ All major and minor version changes will be documented in this file. Details of
 [#468]: https://github.com/gchq/CyberChef/pull/468
 [#476]: https://github.com/gchq/CyberChef/pull/476
 [#489]: https://github.com/gchq/CyberChef/pull/489
+[#496]: https://github.com/gchq/CyberChef/pull/496
 [#506]: https://github.com/gchq/CyberChef/pull/506
 [#516]: https://github.com/gchq/CyberChef/pull/516
+[#525]: https://github.com/gchq/CyberChef/pull/525
+[#533]: https://github.com/gchq/CyberChef/pull/533
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -4,7 +4,6 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
-const Inliner = require("web-resource-inliner");
 const glob = require("glob");
 const path = require("path");
 
@@ -43,18 +42,16 @@ module.exports = function (grunt) {
 
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+        [
+            "eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web",
+            "copy:standalone", "zip:standalone", "clean:standalone", "chmod"
+        ]);
 
     grunt.registerTask("default",
         "Lints the code base",
         ["eslint", "exec:repoSize"]);
 
-    grunt.registerTask("inline",
-        "Compiles a production build of CyberChef into a single, portable web page.",
-        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);
 
-
-    grunt.registerTask("runInliner", runInliner);
     grunt.registerTask("doc", "docs");
     grunt.registerTask("tests", "test");
     grunt.registerTask("lint", "eslint");
@@ -72,6 +69,7 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
     grunt.loadNpmTasks("grunt-contrib-connect");
+    grunt.loadNpmTasks("grunt-zip");
 
 
     // Project configuration
@@ -94,32 +92,6 @@ module.exports = function (grunt) {
         },
         moduleEntryPoints = listEntryModules();
 
-    /**
-     * Compiles a production build of CyberChef into a single, portable web page.
-     */
-    function runInliner() {
-        const done = this.async();
-        Inliner.html({
-            relativeTo: "build/prod/",
-            fileContent: grunt.file.read("build/prod/cyberchef.htm"),
-            images: true,
-            svgs: true,
-            scripts: true,
-            links: true,
-            strict: true
-        }, function(error, result) {
-            if (error) {
-                if (error instanceof Error) {
-                    done(error);
-                } else {
-                    done(new Error(error));
-                }
-            } else {
-                grunt.file.write("build/prod/cyberchef.htm", result);
-                done(true);
-            }
-        });
-    }
 
     /**
      * Generates an entry list for all the modules.
@@ -130,7 +102,7 @@ module.exports = function (grunt) {
         glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
             const basename = path.basename(file);
             if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
-                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
+                entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
         });
 
         return entryModules;
@@ -143,15 +115,15 @@ module.exports = function (grunt) {
             node: ["build/node/*"],
             config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
-            inlineScripts: ["build/prod/scripts.js"],
+            standalone: ["build/prod/CyberChef*.html"]
         },
         eslint: {
             options: {
                 configFile: "./.eslintrc.json"
             },
             configs: ["*.{js,mjs}"],
             core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
-            web: ["src/web/**/*.{js,mjs}"],
+            web: ["src/web/**/*.{js,mjs}", "!src/web/static/**/*"],
             node: ["src/node/**/*.{js,mjs}"],
             tests: ["tests/**/*.{js,mjs}"],
         },
@@ -195,6 +167,9 @@ module.exports = function (grunt) {
                     }, moduleEntryPoints),
                     output: {
                         path: __dirname + "/build/prod",
+                        filename: chunkData => {
+                            return chunkData.chunk.name === "main" ? "assets/[name].js": "[name].js";
+                        },
                         globalObject: "this"
                     },
                     resolve: {
@@ -225,33 +200,6 @@ module.exports = function (grunt) {
                     ]
                 };
             },
-            webInline: {
-                mode: "production",
-                target: "web",
-                entry: "./src/web/index.js",
-                output: {
-                    filename: "scripts.js",
-                    path: __dirname + "/build/prod"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
-                        INLINE: "true"
-                    })),
-                    new HtmlWebpackPlugin({
-                        filename: "cyberchef.htm",
-                        template: "./src/web/html/index.html",
-                        compileTime: compileTime,
-                        version: pkg.version + "s",
-                        inline: true,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
-                        }
-                    }),
-                ]
-            },
             node: {
                 mode: "production",
                 target: "node",
@@ -284,7 +232,8 @@ module.exports = function (grunt) {
                     warningsFilter: [
                         /source-map/,
                         /dependency is an expression/,
-                        /export 'default'/
+                        /export 'default'/,
+                        /Can't resolve 'sodium'/
                     ],
                 }
             },
@@ -316,6 +265,18 @@ module.exports = function (grunt) {
                 }
             }
         },
+        zip: {
+            standalone: {
+                cwd: "build/prod/",
+                src: [
+                    "build/prod/**/*",
+                    "!build/prod/index.html",
+                    "!build/prod/BundleAnalyzerReport.html",
+                    "!build/prod/sitemap.js"
+                ],
+                dest: `build/prod/CyberChef_v${pkg.version}.zip`
+            }
+        },
         connect: {
             prod: {
                 options: {
@@ -328,10 +289,16 @@ module.exports = function (grunt) {
             ghPages: {
                 options: {
                     process: function (content, srcpath) {
-                        // Add Google Analytics code to index.html
                         if (srcpath.indexOf("index.html") >= 0) {
+                            // Add Google Analytics code to index.html
                             content = content.replace("</body></html>",
                                 grunt.file.read("src/web/static/ga.html") + "</body></html>");
+
+                            // Add Structured Data for SEO
+                            content = content.replace("</head>",
+                                "<script type='application/ld+json'>" +
+                                JSON.stringify(JSON.parse(grunt.file.read("src/web/static/structuredData.json"))) +
+                                "</script></head>");
                             return grunt.template.process(content, srcpath);
                         } else {
                             return content;
@@ -350,6 +317,28 @@ module.exports = function (grunt) {
                         dest: "build/prod/"
                     },
                 ]
+            },
+            standalone: {
+                options: {
+                    process: function (content, srcpath) {
+                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Replace download link with version number
+                            content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
+                                `<span>Version ${pkg.version}</span>`);
+
+                            return grunt.template.process(content, srcpath);
+                        } else {
+                            return content;
+                        }
+                    },
+                    noProcess: ["**", "!**/*.html"]
+                },
+                files: [
+                    {
+                        src: "build/prod/index.html",
+                        dest: `build/prod/CyberChef_v${pkg.version}.html`
+                    }
+                ]
             }
         },
         chmod: {
@@ -405,7 +394,7 @@ module.exports = function (grunt) {
                 command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
             },
             browserTests: {
-                command: "./node_modules/.bin/nightwatch --env prod,inline"
+                command: "./node_modules/.bin/nightwatch --env prod"
             }
         },
     });

diff --git a/babel.config.js b/babel.config.js
@@ -11,14 +11,22 @@ module.exports = function(api) {
                     "node": "6.5"
                 },
                 "modules": false,
-                "useBuiltIns": "entry"
+                "useBuiltIns": "entry",
+                "corejs": 3
             }]
         ],
         "plugins": [
             "babel-plugin-syntax-dynamic-import",
-            ["babel-plugin-transform-builtin-extend", {
-                "globals": ["Error"]
-            }]
+            [
+                "babel-plugin-transform-builtin-extend", {
+                    "globals": ["Error"]
+                }
+            ],
+            [
+                "@babel/plugin-transform-runtime", {
+                    "regenerator": true
+                }
+            ]
         ]
     };
 };
diff --git a/nightwatch.json b/nightwatch.json
@@ -23,10 +23,6 @@
 
     "prod": {
       "launch_url": "http://localhost:8000/index.html"
-    },
-
-    "inline": {
-      "launch_url": "http://localhost:8000/cyberchef.htm"
     }
 
   }
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		src/core/vendor/**
		src/web/static/clippy_assets/**